From 421d1751d0a1883c387e4b0bec7167053346834c Mon Sep 17 00:00:00 2001 From: No Author Date: Fri, 28 Feb 2003 18:13:00 +0000 Subject: [PATCH 01/41] New repository initialized by cvs2svn. git-svn-id: svn+ssh://rubyforge.org/var/svn/rubygems/trunk@1 3d4018f9-ac1a-0410-99e9-8a154d859a19 From c7ba5bd8afd74d04e82183d07f829439d2c74808 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Suszy=C5=84ski=20Krzysztof?= Date: Tue, 13 Jun 2017 16:51:12 +0200 Subject: [PATCH 02/41] Initial commit From 5351f0b11d4fc83ec4469108889f4bda4bba89db Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 17 Feb 2022 19:08:29 +0530 Subject: [PATCH 03/41] automate pypi release on a tag Signed-off-by: Tushar Goel --- .github/workflows/pypi-release.yml | 36 ++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/pypi-release.yml diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml new file mode 100644 index 0000000..b668b2e --- /dev/null +++ b/.github/workflows/pypi-release.yml @@ -0,0 +1,36 @@ +name: Release uinvers on PyPI + +on: + release: + types: [created] + +jobs: + build-n-publish: + name: Build and publish univers to PyPI + runs-on: ubuntu-20.04 + +steps: + - uses: actions/checkout@master + - name: Set up Python 3.6 + uses: actions/setup-python@v1 + with: + python-version: 3.6 + - name: Install pypa/build + run: >- + python -m + pip install + build + --user + - name: Build a binary wheel and a source tarball + run: >- + python -m + build + --sdist + --wheel + --outdir dist/ + . + - name: Publish distribution 📦 to PyPI + if: startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@master + with: + password: ${{ secrets.PYPI_API_TOKEN }} \ No newline at end of file From 96b6405269f6b06911bf13194f49ba484f641be3 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 18 Feb 2022 13:49:48 +0100 Subject: [PATCH 04/41] Refine GH Action definition * Do not use explicit references to Python version and project name in descriptions * Use Python 3.8 as a base * Use only plain ASCII Signed-off-by: Philippe Ombredanne --- .github/workflows/pypi-release.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index b668b2e..33eebd1 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -1,4 +1,4 @@ -name: Release uinvers on PyPI +name: Release library as a PyPI wheel and sdist on tag on: release: @@ -6,15 +6,15 @@ on: jobs: build-n-publish: - name: Build and publish univers to PyPI + name: Build and publish library to PyPI runs-on: ubuntu-20.04 steps: - uses: actions/checkout@master - - name: Set up Python 3.6 + - name: Set up Python uses: actions/setup-python@v1 with: - python-version: 3.6 + python-version: 3.8 - name: Install pypa/build run: >- python -m @@ -29,8 +29,9 @@ steps: --wheel --outdir dist/ . - - name: Publish distribution 📦 to PyPI + - name: Publish distribution to PyPI if: startsWith(github.ref, 'refs/tags') uses: pypa/gh-action-pypi-publish@master with: - password: ${{ secrets.PYPI_API_TOKEN }} \ No newline at end of file + password: ${{ secrets.PYPI_API_TOKEN }} + \ No newline at end of file From 635df78840575777c2509525651c0d18e9771e38 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 18 Feb 2022 13:54:51 +0100 Subject: [PATCH 05/41] Format GH action yaml The indentations were not correct Signed-off-by: Philippe Ombredanne --- .github/workflows/pypi-release.yml | 42 ++++++++++++------------------ 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 33eebd1..cb32987 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -8,30 +8,20 @@ jobs: build-n-publish: name: Build and publish library to PyPI runs-on: ubuntu-20.04 - -steps: - - uses: actions/checkout@master - - name: Set up Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Install pypa/build - run: >- - python -m - pip install - build - --user - - name: Build a binary wheel and a source tarball - run: >- - python -m - build - --sdist - --wheel - --outdir dist/ + steps: + - uses: actions/checkout@master + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Install pypa/build + run: python -m pip install build --user + - name: Build a binary wheel and a source tarball + run: python -m build --sdist --wheel --outdir dist/ . - - name: Publish distribution to PyPI - if: startsWith(github.ref, 'refs/tags') - uses: pypa/gh-action-pypi-publish@master - with: - password: ${{ secrets.PYPI_API_TOKEN }} - \ No newline at end of file + - name: Publish distribution to PyPI + if: startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@master + with: + password: ${{ secrets.PYPI_API_TOKEN }} + From 6bbedddfeef1e10383eef87131a162b7fb43df46 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 18 Feb 2022 13:56:07 +0100 Subject: [PATCH 06/41] Use verbose name for job Signed-off-by: Philippe Ombredanne --- .github/workflows/pypi-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index cb32987..188497e 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -5,7 +5,7 @@ on: types: [created] jobs: - build-n-publish: + build-and-publish-to-pypi: name: Build and publish library to PyPI runs-on: ubuntu-20.04 steps: From ad17a42320a8c9e8fd949f10c7b6d0019d035c24 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 23 Feb 2022 22:06:54 +0530 Subject: [PATCH 07/41] Deprecate windows-2016 images for azure CI * Modifies the azure CI for `vs2017-win2016` to `windows-2022` as the former will be deprecated very soon. Signed-off-by: Ayan Sinha Mahapatra --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7cd3025..5df8a18 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -41,8 +41,8 @@ jobs: - template: etc/ci/azure-win.yml parameters: - job_name: win2016_cpython - image_name: vs2017-win2016 + job_name: win2022_cpython + image_name: windows-2022 python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] test_suites: all: venv\Scripts\pytest -n 2 -vvs From cad31644f17aa0af65b5c26c01cdb282f9db0951 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 1 Mar 2022 12:45:59 -0800 Subject: [PATCH 08/41] Remove macos 10.14 job from azure-pipelines.yml Signed-off-by: Jono Yang --- azure-pipelines.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5df8a18..cf057b8 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -23,14 +23,6 @@ jobs: test_suites: all: venv/bin/pytest -n 2 -vvs - - template: etc/ci/azure-posix.yml - parameters: - job_name: macos1014_cpython - image_name: macos-10.14 - python_versions: ['3.6', '3.7', '3.8', '3.9'] - test_suites: - all: venv/bin/pytest -n 2 -vvs - - template: etc/ci/azure-posix.yml parameters: job_name: macos1015_cpython From d659e09be2ff73599d951350d17d4e5c7b72c80c Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 1 Mar 2022 13:10:59 -0800 Subject: [PATCH 09/41] Do not use Python 3.6 on Windows 2022 jobs * Python 3.6 is not available on Windows 2022 images Signed-off-by: Jono Yang --- azure-pipelines.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index cf057b8..f3fd2c3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -33,16 +33,16 @@ jobs: - template: etc/ci/azure-win.yml parameters: - job_name: win2022_cpython - image_name: windows-2022 + job_name: win2019_cpython + image_name: windows-2019 python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] test_suites: all: venv\Scripts\pytest -n 2 -vvs - template: etc/ci/azure-win.yml parameters: - job_name: win2019_cpython - image_name: windows-2019 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + job_name: win2022_cpython + image_name: windows-2022 + python_versions: ['3.7', '3.8', '3.9', '3.10'] test_suites: all: venv\Scripts\pytest -n 2 -vvs From c5251f4762d43cb88ff02636759ff7df991ead05 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 4 Mar 2022 08:22:31 +0100 Subject: [PATCH 10/41] Run tests on macOS 11 Signed-off-by: Philippe Ombredanne --- azure-pipelines.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f3fd2c3..089abe9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -31,6 +31,14 @@ jobs: test_suites: all: venv/bin/pytest -n 2 -vvs + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos11_cpython + image_name: macos-11 + python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + test_suites: + all: venv/bin/pytest -n 2 -vvs + - template: etc/ci/azure-win.yml parameters: job_name: win2019_cpython From a118fe76e3b20a778803d4630222dbf7801c30ae Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 4 Mar 2022 08:53:32 +0100 Subject: [PATCH 11/41] Align configuration scripts on POSIX and Windows Doing so helps with keeping them in sync Signed-off-by: Philippe Ombredanne --- configure | 156 +++++++++++++++++++++++++++----------------------- configure.bat | 17 ++++-- 2 files changed, 97 insertions(+), 76 deletions(-) diff --git a/configure b/configure index fdfdc85..c1d36aa 100755 --- a/configure +++ b/configure @@ -16,6 +16,8 @@ set -e # Source this script for initial configuration # Use configure --help for details # +# NOTE: please keep in sync with Windows script configure.bat +# # This script will search for a virtualenv.pyz app in etc/thirdparty/virtualenv.pyz # Otherwise it will download the latest from the VIRTUALENV_PYZ_URL default ################################ @@ -32,10 +34,8 @@ DEV_REQUIREMENTS="--editable .[testing] --constraint requirements.txt --constrai # where we create a virtualenv VIRTUALENV_DIR=venv -# Cleanable files and directories with the --clean option -CLEANABLE=" - build - venv" +# Cleanable files and directories to delete with the --clean option +CLEANABLE="build venv" # extra arguments passed to pip PIP_EXTRA_ARGS=" " @@ -50,11 +50,14 @@ VIRTUALENV_PYZ_URL=https://bootstrap.pypa.io/virtualenv.pyz CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin + +################################ +# Thirdparty package locations and index handling # Find packages from the local thirdparty directory or from thirdparty.aboutcode.org if [ -f "$CFG_ROOT_DIR/thirdparty" ]; then PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty " fi -PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS --find-links https://thirdparty.aboutcode.org/pypi" +PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS --find-links https://thirdparty.aboutcode.org/pypi/simple/links.html" ################################ @@ -65,56 +68,50 @@ fi ################################ -# find a proper Python to run -# Use environment variables or a file if available. -# Otherwise the latest Python by default. -if [[ "$PYTHON_EXECUTABLE" == "" ]]; then - # check for a file named PYTHON_EXECUTABLE - if [ -f "$CFG_ROOT_DIR/PYTHON_EXECUTABLE" ]; then - PYTHON_EXECUTABLE=$(cat "$CFG_ROOT_DIR/PYTHON_EXECUTABLE") - else - PYTHON_EXECUTABLE=python3 - fi -fi - +# Main command line entry point +main() { + CFG_REQUIREMENTS=$REQUIREMENTS + NO_INDEX="--no-index" + + # We are using getopts to parse option arguments that start with "-" + while getopts :-: optchar; do + case "${optchar}" in + -) + case "${OPTARG}" in + help ) cli_help;; + clean ) find_python && clean;; + dev ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS";; + init ) NO_INDEX="";; + esac;; + esac + done -################################ -cli_help() { - echo An initial configuration script - echo " usage: ./configure [options]" - echo - echo The default is to configure for regular use. Use --dev for development. - echo Use the --init option if starting a new project and the project - echo dependencies are not available on thirdparty.aboutcode.org/pypi/ - echo and requirements.txt and/or requirements-dev.txt has not been generated. - echo - echo The options are: - echo " --clean: clean built and installed files and exit." - echo " --dev: configure the environment for development." - echo " --init: pull dependencies from PyPI. Used when first setting up a project." - echo " --help: display this help message and exit." - echo - echo By default, the python interpreter version found in the path is used. - echo Alternatively, the PYTHON_EXECUTABLE environment variable can be set to - echo configure another Python executable interpreter to use. If this is not - echo set, a file named PYTHON_EXECUTABLE containing a single line with the - echo path of the Python executable to use will be checked last. - set +e - exit + PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS $NO_INDEX" + + find_python + create_virtualenv "$VIRTUALENV_DIR" + install_packages "$CFG_REQUIREMENTS" + . "$CFG_BIN_DIR/activate" } -clean() { - # Remove cleanable file and directories and files from the root dir. - echo "* Cleaning ..." - for cln in $CLEANABLE; - do rm -rf "${CFG_ROOT_DIR:?}/${cln:?}"; - done - set +e - exit +################################ +# Find a proper Python to run +# Use environment variables or a file if available. +# Otherwise the latest Python by default. +find_python() { + if [[ "$PYTHON_EXECUTABLE" == "" ]]; then + # check for a file named PYTHON_EXECUTABLE + if [ -f "$CFG_ROOT_DIR/PYTHON_EXECUTABLE" ]; then + PYTHON_EXECUTABLE=$(cat "$CFG_ROOT_DIR/PYTHON_EXECUTABLE") + else + PYTHON_EXECUTABLE=python3 + fi + fi } +################################ create_virtualenv() { # create a virtualenv for Python # Note: we do not use the bundled Python 3 "venv" because its behavior and @@ -145,6 +142,7 @@ create_virtualenv() { } +################################ install_packages() { # install requirements in virtualenv # note: --no-build-isolation means that pip/wheel/setuptools will not @@ -162,28 +160,44 @@ install_packages() { ################################ -# Main command line entry point -CFG_DEV_MODE=0 -CFG_REQUIREMENTS=$REQUIREMENTS -NO_INDEX="--no-index" - -# We are using getopts to parse option arguments that start with "-" -while getopts :-: optchar; do - case "${optchar}" in - -) - case "${OPTARG}" in - help ) cli_help;; - clean ) clean;; - dev ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS" && CFG_DEV_MODE=1;; - init ) NO_INDEX="";; - esac;; - esac -done - -PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS $NO_INDEX" - -create_virtualenv "$VIRTUALENV_DIR" -install_packages "$CFG_REQUIREMENTS" -. "$CFG_BIN_DIR/activate" +cli_help() { + echo An initial configuration script + echo " usage: ./configure [options]" + echo + echo The default is to configure for regular use. Use --dev for development. + echo Use the --init option if starting a new project and the project + echo dependencies are not available on thirdparty.aboutcode.org/pypi/ + echo and requirements.txt and/or requirements-dev.txt has not been generated. + echo + echo The options are: + echo " --clean: clean built and installed files and exit." + echo " --dev: configure the environment for development." + echo " --init: pull dependencies from PyPI. Used when first setting up a project." + echo " --help: display this help message and exit." + echo + echo By default, the python interpreter version found in the path is used. + echo Alternatively, the PYTHON_EXECUTABLE environment variable can be set to + echo configure another Python executable interpreter to use. If this is not + echo set, a file named PYTHON_EXECUTABLE containing a single line with the + echo path of the Python executable to use will be checked last. + set +e + exit +} + + +################################ +clean() { + # Remove cleanable file and directories and files from the root dir. + echo "* Cleaning ..." + for cln in $CLEANABLE; + do rm -rf "${CFG_ROOT_DIR:?}/${cln:?}"; + done + set +e + exit +} + + + +main set +e diff --git a/configure.bat b/configure.bat index ed06161..961e0d9 100644 --- a/configure.bat +++ b/configure.bat @@ -14,6 +14,8 @@ @rem # Source this script for initial configuration @rem # Use configure --help for details +@rem # NOTE: please keep in sync with POSIX script configure + @rem # This script will search for a virtualenv.pyz app in etc\thirdparty\virtualenv.pyz @rem # Otherwise it will download the latest from the VIRTUALENV_PYZ_URL default @rem ################################ @@ -49,10 +51,11 @@ set "CFG_BIN_DIR=%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts" @rem ################################ @rem # Thirdparty package locations and index handling +@rem # Find packages from the local thirdparty directory or from thirdparty.aboutcode.org if exist "%CFG_ROOT_DIR%\thirdparty" ( set PIP_EXTRA_ARGS=--find-links "%CFG_ROOT_DIR%\thirdparty" ) -set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% --find-links https://thirdparty.aboutcode.org/pypi" & %INDEX_ARG% +set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% --find-links https://thirdparty.aboutcode.org/pypi/simple/links.html" @rem ################################ @@ -64,7 +67,6 @@ if not defined CFG_QUIET ( @rem ################################ @rem # Main command line entry point -set CFG_DEV_MODE=0 set "CFG_REQUIREMENTS=%REQUIREMENTS%" set "NO_INDEX=--no-index" @@ -74,7 +76,6 @@ if not "%1" == "" ( if "%1" EQU "--clean" (goto clean) if "%1" EQU "--dev" ( set "CFG_REQUIREMENTS=%DEV_REQUIREMENTS%" - set CFG_DEV_MODE=1 ) if "%1" EQU "--init" ( set "NO_INDEX= " @@ -87,7 +88,7 @@ set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% %NO_INDEX%" @rem ################################ -@rem # find a proper Python to run +@rem # Find a proper Python to run @rem # Use environment variables or a file if available. @rem # Otherwise the latest Python by default. if not defined PYTHON_EXECUTABLE ( @@ -99,6 +100,8 @@ if not defined PYTHON_EXECUTABLE ( ) ) + +@rem ################################ :create_virtualenv @rem # create a virtualenv for Python @rem # Note: we do not use the bundled Python 3 "venv" because its behavior and @@ -143,6 +146,7 @@ if %ERRORLEVEL% neq 0 ( ) +@rem ################################ :install_packages @rem # install requirements in virtualenv @rem # note: --no-build-isolation means that pip/wheel/setuptools will not @@ -157,6 +161,9 @@ if %ERRORLEVEL% neq 0 ( %PIP_EXTRA_ARGS% ^ %CFG_REQUIREMENTS% + +@rem ################################ +:create_bin_junction @rem # Create junction to bin to have the same directory between linux and windows if exist "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin" ( rmdir /s /q "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin" @@ -171,7 +178,6 @@ exit /b 0 @rem ################################ - :cli_help echo An initial configuration script echo " usage: configure [options]" @@ -195,6 +201,7 @@ exit /b 0 exit /b 0 +@rem ################################ :clean @rem # Remove cleanable file and directories and files from the root dir. echo "* Cleaning ..." From e810da356b99cb7241c90c9a79b20232ddebbb50 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 4 Mar 2022 09:00:46 +0100 Subject: [PATCH 12/41] Update README Signed-off-by: Philippe Ombredanne --- README.rst | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 4173689..74e58fa 100644 --- a/README.rst +++ b/README.rst @@ -1,19 +1,27 @@ A Simple Python Project Skeleton ================================ -This repo attempts to standardize our python repositories using modern python -packaging and configuration techniques. Using this `blog post`_ as inspiration, this -repository will serve as the base for all new python projects and will be adopted to all -our existing ones as well. +This repo attempts to standardize the structure of the Python-based project's +repositories using modern Python packaging and configuration techniques. +Using this `blog post`_ as inspiration, this repository serves as the base for +all new Python projects and is mergeable in existing repositories as well. .. _blog post: https://blog.jaraco.com/a-project-skeleton-for-python-projects/ + Usage ===== Usage instructions can be found in ``docs/skeleton-usage.rst``. + Release Notes ============= +- 2022-03-04: + - Synchronize configure and configure.bat scripts for sanity + - Update CI operating system support with latest Azure OS images + - Streamline utility scripts in etc/scripts/ to create, fetch and manage third-party dependencies + There are now fewer scripts. See etc/scripts/README.rst for details + - 2021-09-03: - ``configure`` now requires pinned dependencies via the use of ``requirements.txt`` and ``requirements-dev.txt`` - ``configure`` can now accept multiple options at once From 243f7cb96ec9ab03080ffb8e69197b80894ef565 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 5 Mar 2022 07:07:18 +0100 Subject: [PATCH 13/41] Refactor and streamline thirdparty utilities There is now a single primary script "fetch_thirdparty.py" that handles everything in a simplified way. There is no utility to build wheels anymore: these MUST be available before hand on PyPI or built manually and uploaded in our self-hosted PyPI repository. Signed-off-by: Philippe Ombredanne --- etc/scripts/README.rst | 45 +- etc/scripts/bootstrap.py | 244 ---- etc/scripts/build_wheels.py | 121 -- etc/scripts/check_thirdparty.py | 30 +- etc/scripts/fetch_built_wheels.py | 63 - etc/scripts/fetch_requirements.py | 159 --- etc/scripts/fetch_thirdparty.py | 306 +++++ etc/scripts/fix_thirdparty.py | 114 -- etc/scripts/gen_pypi_simple.py | 246 +++- etc/scripts/publish_files.py | 208 --- etc/scripts/utils_requirements.py | 147 +- etc/scripts/utils_thirdparty.py | 2079 +++++++++-------------------- 12 files changed, 1233 insertions(+), 2529 deletions(-) delete mode 100644 etc/scripts/bootstrap.py delete mode 100644 etc/scripts/build_wheels.py delete mode 100644 etc/scripts/fetch_built_wheels.py delete mode 100644 etc/scripts/fetch_requirements.py create mode 100644 etc/scripts/fetch_thirdparty.py delete mode 100644 etc/scripts/fix_thirdparty.py delete mode 100644 etc/scripts/publish_files.py diff --git a/etc/scripts/README.rst b/etc/scripts/README.rst index d8b00f9..edf82e4 100755 --- a/etc/scripts/README.rst +++ b/etc/scripts/README.rst @@ -15,10 +15,10 @@ Pre-requisites * To generate or update pip requirement files, you need to start with a clean virtualenv as instructed below (This is to avoid injecting requirements - specific to the tools here in the main requirements). + specific to the tools used here in the main requirements). * For other usages, the tools here can run either in their own isolated - virtualenv best or in the the main configured development virtualenv. + virtualenv or in the the main configured development virtualenv. These requireements need to be installed:: pip install --requirement etc/release/requirements.txt @@ -82,45 +82,14 @@ Populate a thirdparty directory with wheels, sources, .ABOUT and license files Scripts ~~~~~~~ -* **fetch_requirements.py** will fetch package wheels, their ABOUT, LICENSE and - NOTICE files to populate a local a thirdparty directory strictly from our - remote repo and using only pinned packages listed in one or more pip - requirements file(s). Fetch only requirements for specific python versions and - operating systems. Optionally fetch the corresponding source distributions. - -* **publish_files.py** will upload/sync a thirdparty directory of files to our - remote repo. Requires a GitHub personal access token. - -* **build_wheels.py** will build a package binary wheel for multiple OS and - python versions. Optionally wheels that contain native code are built - remotely. Dependent wheels are optionally included. Requires Azure credentials - and tokens if building wheels remotely on multiple operatin systems. - -* **fix_thirdparty.py** will fix a thirdparty directory with a best effort to - add missing wheels, sources archives, create or fetch or fix .ABOUT, .NOTICE - and .LICENSE files. Requires Azure credentials and tokens if requesting the - build of missing wheels remotely on multiple operatin systems. +* **fetch_thirdparty.py** will fetch package wheels, source sdist tarballs + and their ABOUT, LICENSE and NOTICE files to populate a local directory from + a list of PyPI simple URLs (typically PyPI.org proper and our self-hosted PyPI) + using pip requirements file(s), specifiers or pre-existing packages files. + Fetch wheels for specific python version and operating system combinations. * **check_thirdparty.py** will check a thirdparty directory for errors. -* **bootstrap.py** will bootstrap a thirdparty directory from a requirements - file(s) to add or build missing wheels, sources archives and create .ABOUT, - .NOTICE and .LICENSE files. Requires Azure credentials and tokens if - requesting the build of missing wheels remotely on multiple operatin systems. - - - -Usage -~~~~~ - -See each command line --help option for details. - -* (TODO) **add_package.py** will add or update a Python package including wheels, - sources and ABOUT files and this for multiple Python version and OSes(for use - with upload_packages.py afterwards) You will need an Azure personal access - token for buidling binaries and an optional DejaCode API key to post and fetch - new package versions there. TODO: explain how we use romp - Upgrade virtualenv app ---------------------- diff --git a/etc/scripts/bootstrap.py b/etc/scripts/bootstrap.py deleted file mode 100644 index 31f2f55..0000000 --- a/etc/scripts/bootstrap.py +++ /dev/null @@ -1,244 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import itertools - -import click - -import utils_thirdparty -from utils_thirdparty import Environment -from utils_thirdparty import PypiPackage - - -@click.command() -@click.option( - "-r", - "--requirements-file", - type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False), - metavar="FILE", - multiple=True, - default=["requirements.txt"], - show_default=True, - help="Path to the requirements file(s) to use for thirdparty packages.", -) -@click.option( - "-d", - "--thirdparty-dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), - metavar="DIR", - default=utils_thirdparty.THIRDPARTY_DIR, - show_default=True, - help="Path to the thirdparty directory where wheels are built and " - "sources, ABOUT and LICENSE files fetched.", -) -@click.option( - "-p", - "--python-version", - type=click.Choice(utils_thirdparty.PYTHON_VERSIONS), - metavar="PYVER", - default=utils_thirdparty.PYTHON_VERSIONS, - show_default=True, - multiple=True, - help="Python version(s) to use for this build.", -) -@click.option( - "-o", - "--operating-system", - type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS), - metavar="OS", - default=tuple(utils_thirdparty.PLATFORMS_BY_OS), - multiple=True, - show_default=True, - help="OS(ses) to use for this build: one of linux, mac or windows.", -) -@click.option( - "-l", - "--latest-version", - is_flag=True, - help="Get the latest version of all packages, ignoring version specifiers.", -) -@click.option( - "--sync-dejacode", - is_flag=True, - help="Synchronize packages with DejaCode.", -) -@click.option( - "--with-deps", - is_flag=True, - help="Also include all dependent wheels.", -) -@click.help_option("-h", "--help") -def bootstrap( - requirements_file, - thirdparty_dir, - python_version, - operating_system, - with_deps, - latest_version, - sync_dejacode, - build_remotely=False, -): - """ - Boostrap a thirdparty Python packages directory from pip requirements. - - Fetch or build to THIRDPARTY_DIR all the wheels and source distributions for - the pip ``--requirement-file`` requirements FILE(s). Build wheels compatible - with all the provided ``--python-version`` PYVER(s) and ```--operating_system`` - OS(s) defaulting to all supported combinations. Create or fetch .ABOUT and - .LICENSE files. - - Optionally ignore version specifiers and use the ``--latest-version`` - of everything. - - Sources and wheels are fetched with attempts first from PyPI, then our remote repository. - If missing wheels are built as needed. - """ - # rename variables for clarity since these are lists - requirements_files = requirements_file - python_versions = python_version - operating_systems = operating_system - - # create the environments we need - evts = itertools.product(python_versions, operating_systems) - environments = [Environment.from_pyver_and_os(pyv, os) for pyv, os in evts] - - # collect all packages to process from requirements files - # this will fail with an exception if there are packages we cannot find - - required_name_versions = set() - - for req_file in requirements_files: - nvs = utils_thirdparty.load_requirements(requirements_file=req_file, force_pinned=False) - required_name_versions.update(nvs) - if latest_version: - required_name_versions = set((name, None) for name, _ver in required_name_versions) - - print( - f"PROCESSING {len(required_name_versions)} REQUIREMENTS in {len(requirements_files)} FILES" - ) - - # fetch all available wheels, keep track of missing - # start with local, then remote, then PyPI - - print("==> COLLECTING ALREADY LOCALLY AVAILABLE REQUIRED WHEELS") - # list of all the wheel filenames either pre-existing, fetched or built - # updated as we progress - available_wheel_filenames = [] - - local_packages_by_namever = { - (p.name, p.version): p - for p in utils_thirdparty.get_local_packages(directory=thirdparty_dir) - } - - # list of (name, version, environment) not local and to fetch - name_version_envt_to_fetch = [] - - # start with a local check - for (name, version), envt in itertools.product(required_name_versions, environments): - local_pack = local_packages_by_namever.get( - ( - name, - version, - ) - ) - if local_pack: - supported_wheels = list(local_pack.get_supported_wheels(environment=envt)) - if supported_wheels: - available_wheel_filenames.extend(w.filename for w in supported_wheels) - print( - f"====> No fetch or build needed. " - f"Local wheel already available for {name}=={version} " - f"on os: {envt.operating_system} for Python: {envt.python_version}" - ) - continue - - name_version_envt_to_fetch.append( - ( - name, - version, - envt, - ) - ) - - print(f"==> TRYING TO FETCH #{len(name_version_envt_to_fetch)} REQUIRED WHEELS") - - # list of (name, version, environment) not fetch and to build - name_version_envt_to_build = [] - - # then check if the wheel can be fetched without building from remote and Pypi - for name, version, envt in name_version_envt_to_fetch: - - fetched_fwn = utils_thirdparty.fetch_package_wheel( - name=name, - version=version, - environment=envt, - dest_dir=thirdparty_dir, - ) - - if fetched_fwn: - available_wheel_filenames.append(fetched_fwn) - else: - name_version_envt_to_build.append( - ( - name, - version, - envt, - ) - ) - - # At this stage we have all the wheels we could obtain without building - for name, version, envt in name_version_envt_to_build: - print( - f"====> Need to build wheels for {name}=={version} on os: " - f"{envt.operating_system} for Python: {envt.python_version}" - ) - - packages_and_envts_to_build = [ - (PypiPackage(name, version), envt) for name, version, envt in name_version_envt_to_build - ] - - print(f"==> BUILDING #{len(packages_and_envts_to_build)} MISSING WHEELS") - - package_envts_not_built, wheel_filenames_built = utils_thirdparty.build_missing_wheels( - packages_and_envts=packages_and_envts_to_build, - build_remotely=build_remotely, - with_deps=with_deps, - dest_dir=thirdparty_dir, - ) - if wheel_filenames_built: - available_wheel_filenames.extend(available_wheel_filenames) - - for pack, envt in package_envts_not_built: - print( - f"====> FAILED to build any wheel for {pack.name}=={pack.version} " - f"on os: {envt.operating_system} for Python: {envt.python_version}" - ) - - print(f"==> FETCHING SOURCE DISTRIBUTIONS") - # fetch all sources, keep track of missing - # This is a list of (name, version) - utils_thirdparty.fetch_missing_sources(dest_dir=thirdparty_dir) - - print(f"==> FETCHING ABOUT AND LICENSE FILES") - utils_thirdparty.add_fetch_or_update_about_and_license_files(dest_dir=thirdparty_dir) - - ############################################################################ - if sync_dejacode: - print(f"==> SYNC WITH DEJACODE") - # try to fetch from DejaCode any missing ABOUT - # create all missing DejaCode packages - pass - - utils_thirdparty.find_problems(dest_dir=thirdparty_dir) - - -if __name__ == "__main__": - bootstrap() diff --git a/etc/scripts/build_wheels.py b/etc/scripts/build_wheels.py deleted file mode 100644 index 8a28176..0000000 --- a/etc/scripts/build_wheels.py +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import click - -import utils_thirdparty - - -@click.command() -@click.option( - "-n", - "--name", - type=str, - metavar="PACKAGE_NAME", - required=True, - help="Python package name to add or build.", -) -@click.option( - "-v", - "--version", - type=str, - default=None, - metavar="VERSION", - help="Python package version to add or build.", -) -@click.option( - "-d", - "--thirdparty-dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), - metavar="DIR", - default=utils_thirdparty.THIRDPARTY_DIR, - show_default=True, - help="Path to the thirdparty directory where wheels are built.", -) -@click.option( - "-p", - "--python-version", - type=click.Choice(utils_thirdparty.PYTHON_VERSIONS), - metavar="PYVER", - default=utils_thirdparty.PYTHON_VERSIONS, - show_default=True, - multiple=True, - help="Python version to use for this build.", -) -@click.option( - "-o", - "--operating-system", - type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS), - metavar="OS", - default=tuple(utils_thirdparty.PLATFORMS_BY_OS), - multiple=True, - show_default=True, - help="OS to use for this build: one of linux, mac or windows.", -) -@click.option( - "--build-remotely", - is_flag=True, - help="Build missing wheels remotely.", -) -@click.option( - "--with-deps", - is_flag=True, - help="Also include all dependent wheels.", -) -@click.option( - "--remote-build-log-file", - type=click.Path(writable=True), - default=None, - metavar="LOG-FILE", - help="Path to an optional log file where to list remote builds download URLs. " - "If provided, do not wait for remote builds to complete (and therefore, " - "do not download them either). Instead create a JSON lines log file with " - "one entry for each build suitable to fetch the artifacts at a later time.", -) -@click.option( - "--verbose", - is_flag=True, - help="Provide verbose output.", -) -@click.help_option("-h", "--help") -def build_wheels( - name, - version, - thirdparty_dir, - python_version, - operating_system, - with_deps, - build_remotely, - remote_build_log_file, - verbose, -): - """ - Build to THIRDPARTY_DIR all the wheels for the Python PACKAGE_NAME and - optional VERSION. Build wheels compatible with all the `--python-version` - PYVER(s) and `--operating_system` OS(s). - - Build native wheels remotely if needed when `--build-remotely` and include - all dependencies with `--with-deps`. - """ - utils_thirdparty.add_or_upgrade_built_wheels( - name=name, - version=version, - python_versions=python_version, - operating_systems=operating_system, - dest_dir=thirdparty_dir, - build_remotely=build_remotely, - with_deps=with_deps, - verbose=verbose, - remote_build_log_file=remote_build_log_file, - ) - - -if __name__ == "__main__": - build_wheels() diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py index 4fea16c..0f04b34 100644 --- a/etc/scripts/check_thirdparty.py +++ b/etc/scripts/check_thirdparty.py @@ -16,17 +16,39 @@ @click.command() @click.option( "-d", - "--thirdparty-dir", + "--dest_dir", type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), required=True, help="Path to the thirdparty directory to check.", ) +@click.option( + "-w", + "--wheels", + is_flag=True, + help="Check missing wheels.", +) +@click.option( + "-s", + "--sdists", + is_flag=True, + help="Check missing source sdists tarballs.", +) @click.help_option("-h", "--help") -def check_thirdparty_dir(thirdparty_dir): +def check_thirdparty_dir( + dest_dir, + wheels, + sdists, +): """ - Check a thirdparty directory for problems. + Check a thirdparty directory for problems and print these on screen. """ - utils_thirdparty.find_problems(dest_dir=thirdparty_dir) + # check for problems + print(f"==> CHECK FOR PROBLEMS") + utils_thirdparty.find_problems( + dest_dir=dest_dir, + report_missing_sources=sdists, + report_missing_wheels=wheels, + ) if __name__ == "__main__": diff --git a/etc/scripts/fetch_built_wheels.py b/etc/scripts/fetch_built_wheels.py deleted file mode 100644 index a78861e..0000000 --- a/etc/scripts/fetch_built_wheels.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/scancode-toolkit for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import click - -import utils_thirdparty - - -@click.command() -@click.option( - "--remote-build-log-file", - type=click.Path(readable=True), - metavar="LOG-FILE", - help="Path to a remote builds log file.", -) -@click.option( - "-d", - "--thirdparty-dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), - metavar="DIR", - default=utils_thirdparty.THIRDPARTY_DIR, - show_default=True, - help="Path to the thirdparty directory to save built wheels.", -) -@click.option( - "--no-wait", - is_flag=True, - default=False, - help="Do not wait for build completion.", -) -@click.option( - "--verbose", - is_flag=True, - help="Provide verbose output.", -) -@click.help_option("-h", "--help") -def fetch_remote_wheels( - remote_build_log_file, - thirdparty_dir, - no_wait, - verbose, -): - """ - Fetch to THIRDPARTY_DIR all the wheels built in the LOG-FILE JSON lines - build log file. - """ - utils_thirdparty.fetch_remotely_built_wheels( - remote_build_log_file=remote_build_log_file, - dest_dir=thirdparty_dir, - no_wait=no_wait, - verbose=verbose, - ) - - -if __name__ == "__main__": - fetch_remote_wheels() diff --git a/etc/scripts/fetch_requirements.py b/etc/scripts/fetch_requirements.py deleted file mode 100644 index 9da9ce9..0000000 --- a/etc/scripts/fetch_requirements.py +++ /dev/null @@ -1,159 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import itertools - -import click - -import utils_thirdparty - - -@click.command() -@click.option( - "-r", - "--requirements-file", - type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False), - metavar="FILE", - multiple=True, - default=["requirements.txt"], - show_default=True, - help="Path to the requirements file to use for thirdparty packages.", -) -@click.option( - "-d", - "--thirdparty-dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), - metavar="DIR", - default=utils_thirdparty.THIRDPARTY_DIR, - show_default=True, - help="Path to the thirdparty directory.", -) -@click.option( - "-p", - "--python-version", - type=click.Choice(utils_thirdparty.PYTHON_VERSIONS), - metavar="INT", - multiple=True, - default=["36"], - show_default=True, - help="Python version to use for this build.", -) -@click.option( - "-o", - "--operating-system", - type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS), - metavar="OS", - multiple=True, - default=["linux"], - show_default=True, - help="OS to use for this build: one of linux, mac or windows.", -) -@click.option( - "-s", - "--with-sources", - is_flag=True, - help="Fetch the corresponding source distributions.", -) -@click.option( - "-a", - "--with-about", - is_flag=True, - help="Fetch the corresponding ABOUT and LICENSE files.", -) -@click.option( - "--allow-unpinned", - is_flag=True, - help="Allow requirements without pinned versions.", -) -@click.option( - "-s", - "--only-sources", - is_flag=True, - help="Fetch only the corresponding source distributions.", -) -@click.option( - "-u", - "--remote-links-url", - type=str, - metavar="URL", - default=utils_thirdparty.REMOTE_LINKS_URL, - show_default=True, - help="URL to a PyPI-like links web site. " "Or local path to a directory with wheels.", -) -@click.help_option("-h", "--help") -def fetch_requirements( - requirements_file, - thirdparty_dir, - python_version, - operating_system, - with_sources, - with_about, - allow_unpinned, - only_sources, - remote_links_url=utils_thirdparty.REMOTE_LINKS_URL, -): - """ - Fetch and save to THIRDPARTY_DIR all the required wheels for pinned - dependencies found in the `--requirement` FILE requirements file(s). Only - fetch wheels compatible with the provided `--python-version` and - `--operating-system`. - Also fetch the corresponding .ABOUT, .LICENSE and .NOTICE files together - with a virtualenv.pyz app. - - Use exclusively wheel not from PyPI but rather found in the PyPI-like link - repo ``remote_links_url`` if this is a URL. Treat this ``remote_links_url`` - as a local directory path to a wheels directory if this is not a a URL. - """ - - # fetch wheels - python_versions = python_version - operating_systems = operating_system - requirements_files = requirements_file - - if not only_sources: - envs = itertools.product(python_versions, operating_systems) - envs = (utils_thirdparty.Environment.from_pyver_and_os(pyv, os) for pyv, os in envs) - - for env, reqf in itertools.product(envs, requirements_files): - - for package, error in utils_thirdparty.fetch_wheels( - environment=env, - requirements_file=reqf, - allow_unpinned=allow_unpinned, - dest_dir=thirdparty_dir, - remote_links_url=remote_links_url, - ): - if error: - print("Failed to fetch wheel:", package, ":", error) - - # optionally fetch sources - if with_sources or only_sources: - - for reqf in requirements_files: - for package, error in utils_thirdparty.fetch_sources( - requirements_file=reqf, - allow_unpinned=allow_unpinned, - dest_dir=thirdparty_dir, - remote_links_url=remote_links_url, - ): - if error: - print("Failed to fetch source:", package, ":", error) - - if with_about: - utils_thirdparty.add_fetch_or_update_about_and_license_files(dest_dir=thirdparty_dir) - utils_thirdparty.find_problems( - dest_dir=thirdparty_dir, - report_missing_sources=with_sources or only_sources, - report_missing_wheels=not only_sources, - ) - - -if __name__ == "__main__": - fetch_requirements() diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py new file mode 100644 index 0000000..22147b2 --- /dev/null +++ b/etc/scripts/fetch_thirdparty.py @@ -0,0 +1,306 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/skeleton for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import itertools +import os +import sys + +import click + +import utils_thirdparty +import utils_requirements + +TRACE = True + + +@click.command() +@click.option( + "-r", + "--requirements", + "requirements_files", + type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False), + metavar="REQUIREMENT-FILE", + multiple=True, + required=False, + help="Path to pip requirements file(s) listing thirdparty packages.", +) +@click.option( + "--spec", + "--specifier", + "specifiers", + type=str, + metavar="SPECIFIER", + multiple=True, + required=False, + help="Thirdparty package name==version specification(s) as in django==1.2.3. " + "With --latest-version a plain package name is also acceptable.", +) +@click.option( + "-l", + "--latest-version", + is_flag=True, + help="Get the latest version of all packages, ignoring any specified versions.", +) +@click.option( + "-d", + "--dest", + "dest_dir", + type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), + metavar="DIR", + default=utils_thirdparty.THIRDPARTY_DIR, + show_default=True, + help="Path to the detsination directory where to save downloaded wheels, " + "sources, ABOUT and LICENSE files..", +) +@click.option( + "-w", + "--wheels", + is_flag=True, + help="Download wheels.", +) +@click.option( + "-s", + "--sdists", + is_flag=True, + help="Download source sdists tarballs.", +) +@click.option( + "-p", + "--python-version", + "python_versions", + type=click.Choice(utils_thirdparty.PYTHON_VERSIONS), + metavar="PYVER", + default=utils_thirdparty.PYTHON_VERSIONS, + show_default=True, + multiple=True, + help="Python version(s) to use for wheels.", +) +@click.option( + "-o", + "--operating-system", + "operating_systems", + type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS), + metavar="OS", + default=tuple(utils_thirdparty.PLATFORMS_BY_OS), + multiple=True, + show_default=True, + help="OS(ses) to use for wheels: one of linux, mac or windows.", +) +@click.option( + "--index-url", + "index_urls", + type=str, + metavar="INDEX", + default=utils_thirdparty.PYPI_INDEXES, + show_default=True, + multiple=True, + help="PyPI index URL(s) to use for wheels and sources, in order of preferences.", +) +@click.help_option("-h", "--help") +def fetch_thirdparty( + requirements_files, + specifiers, + latest_version, + dest_dir, + python_versions, + operating_systems, + wheels, + sdists, + index_urls, +): + """ + Download to --dest-dir THIRDPARTY_DIR the PyPI wheels, source distributions, + and their ABOUT metadata, license and notices files. + + Download the PyPI packages listed in the combination of: + - the pip requirements --requirements REQUIREMENT-FILE(s), + - the pip name==version --specifier SPECIFIER(s) + - any pre-existing wheels or sdsists found in --dest-dir THIRDPARTY_DIR. + + Download wheels with the --wheels option for the ``--python-version`` PYVER(s) + and ``--operating_system`` OS(s) combinations defaulting to all supported combinations. + + Download sdists tarballs with the --sdists option. + + Generate or Download .ABOUT, .LICENSE and .NOTICE files for all the wheels and sources fetched. + + Download wheels and sdists the provided PyPI simple --index-url INDEX(s) URLs. + """ + print(f"COLLECTING REQUIRED NAMES & VERSIONS FROM {dest_dir}") + existing_packages_by_nv = { + (package.name, package.version): package + for package in utils_thirdparty.get_local_packages(directory=dest_dir) + } + + required_name_versions = set(existing_packages_by_nv.keys()) + + for req_file in requirements_files: + nvs = utils_requirements.load_requirements( + requirements_file=req_file, + with_unpinned=latest_version, + ) + required_name_versions.update(nvs) + + for specifier in specifiers: + nv = utils_requirements.get_name_version( + requirement=specifier, + with_unpinned=latest_version, + ) + required_name_versions.add(nv) + + if not required_name_versions: + print("Error: no requirements requested.") + sys.exit(1) + + if not os.listdir(dest_dir) and not (wheels or sdists): + print("Error: one or both of --wheels and --sdists is required.") + sys.exit(1) + + if latest_version: + latest_name_versions = set() + names = set(name for name, _version in sorted(required_name_versions)) + for name in sorted(names): + latests = utils_thirdparty.PypiPackage.sorted( + utils_thirdparty.get_package_versions( + name=name, version=None, index_urls=index_urls + ) + ) + if not latests: + print(f"No distribution found for: {name}") + continue + latest = latests[-1] + latest_name_versions.add((latest.name, latest.version)) + required_name_versions = latest_name_versions + + if TRACE: + print("required_name_versions:", required_name_versions) + + if wheels: + # create the environments matrix we need for wheels + evts = itertools.product(python_versions, operating_systems) + environments = [utils_thirdparty.Environment.from_pyver_and_os(pyv, os) for pyv, os in evts] + + wheels_not_found = {} + sdists_not_found = {} + # iterate over requirements, one at a time + for name, version in sorted(required_name_versions): + nv = name, version + existing_package = existing_packages_by_nv.get(nv) + if wheels: + for environment in environments: + if existing_package: + existing_wheels = list( + existing_package.get_supported_wheels(environment=environment) + ) + else: + existing_wheels = None + + if existing_wheels: + if TRACE: + print( + f"====> Wheels already available: {name}=={version} on: {environment}: {existing_package.wheels!r}" + ) + if all(w.is_pure() for w in existing_wheels): + break + else: + continue + + if TRACE: + print(f"Fetching wheel for: {name}=={version} on: {environment}") + + try: + ( + fetched_wheel_filenames, + existing_wheel_filenames, + ) = utils_thirdparty.download_wheel( + name=name, + version=version, + environment=environment, + dest_dir=dest_dir, + index_urls=index_urls, + ) + if TRACE: + if existing_wheel_filenames: + print( + f" ====> Wheels already available: {name}=={version} on: {environment}" + ) + for whl in existing_wheel_filenames: + print(f" {whl}") + if fetched_wheel_filenames: + print(f" ====> Wheels fetched: {name}=={version} on: {environment}") + for whl in fetched_wheel_filenames: + print(f" {whl}") + + fwfns = fetched_wheel_filenames + existing_wheel_filenames + + if all(utils_thirdparty.Wheel.from_filename(f).is_pure() for f in fwfns): + break + + except utils_thirdparty.DistributionNotFound as e: + wheels_not_found[f"{name}=={version}"] = str(e) + + if sdists: + if existing_package and existing_package.sdist: + if TRACE: + print( + f" ====> Sdist already available: {name}=={version}: {existing_package.sdist!r}" + ) + continue + + if TRACE: + print(f" Fetching sdist for: {name}=={version}") + + try: + fetched = utils_thirdparty.download_sdist( + name=name, + version=version, + dest_dir=dest_dir, + index_urls=index_urls, + ) + + if TRACE: + if not fetched: + print( + f" ====> Sdist already available: {name}=={version} on: {environment}" + ) + else: + print( + f" ====> Sdist fetched: {fetched} for {name}=={version} on: {environment}" + ) + + except utils_thirdparty.DistributionNotFound as e: + sdists_not_found[f"{name}=={version}"] = str(e) + + if wheels and wheels_not_found: + print(f"==> MISSING WHEELS") + for wh in wheels_not_found: + print(f" {wh}") + + if sdists and sdists_not_found: + print(f"==> MISSING SDISTS") + for sd in sdists_not_found: + print(f" {sd}") + + print(f"==> FETCHING OR CREATING ABOUT AND LICENSE FILES") + utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir) + utils_thirdparty.clean_about_files(dest_dir=dest_dir) + + # check for problems + print(f"==> CHECK FOR PROBLEMS") + utils_thirdparty.find_problems( + dest_dir=dest_dir, + report_missing_sources=sdists, + report_missing_wheels=wheels, + ) + + +if __name__ == "__main__": + fetch_thirdparty() diff --git a/etc/scripts/fix_thirdparty.py b/etc/scripts/fix_thirdparty.py deleted file mode 100644 index d664c9c..0000000 --- a/etc/scripts/fix_thirdparty.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import click - -import utils_thirdparty - - -@click.command() -@click.option( - "-d", - "--thirdparty-dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), - required=True, - help="Path to the thirdparty directory to fix.", -) -@click.option( - "--build-wheels", - is_flag=True, - help="Build all missing wheels .", -) -@click.option( - "--build-remotely", - is_flag=True, - help="Build missing wheels remotely.", -) -@click.option( - "--remote-build-log-file", - type=click.Path(writable=True), - default=None, - metavar="LOG-FILE", - help="Path to an optional log file where to list remote builds download URLs. " - "If provided, do not wait for remote builds to complete (and therefore, " - "do not download them either). Instead create a JSON lines log file with " - "one entry for each build suitable to fetch the artifacts at a later time.", -) -@click.option( - "--strip-classifiers", - is_flag=True, - help="Remove danglingf classifiers", -) -@click.help_option("-h", "--help") -def fix_thirdparty_dir( - thirdparty_dir, - build_wheels, - build_remotely, - remote_build_log_file, - strip_classifiers, -): - """ - Fix a thirdparty directory of dependent package wheels and sdist. - - Multiple fixes are applied: - - fetch or build missing binary wheels - - fetch missing source distributions - - derive, fetch or add missing ABOUT files - - fetch missing .LICENSE and .NOTICE files - - remove outdated package versions and the ABOUT, .LICENSE and .NOTICE files - - Optionally build missing binary wheels for all supported OS and Python - version combos locally or remotely. - """ - if strip_classifiers: - print("***ADD*** ABOUT AND LICENSES, STRIP CLASSIFIERS") - utils_thirdparty.add_fetch_or_update_about_and_license_files( - dest_dir=thirdparty_dir, - strip_classifiers=strip_classifiers, - ) - else: - print("***FETCH*** MISSING WHEELS") - package_envts_not_fetched = utils_thirdparty.fetch_missing_wheels(dest_dir=thirdparty_dir) - print("***FETCH*** MISSING SOURCES") - src_name_ver_not_fetched = utils_thirdparty.fetch_missing_sources(dest_dir=thirdparty_dir) - - package_envts_not_built = [] - if build_wheels: - print("***BUILD*** MISSING WHEELS") - results = utils_thirdparty.build_missing_wheels( - packages_and_envts=package_envts_not_fetched, - build_remotely=build_remotely, - remote_build_log_file=remote_build_log_file, - dest_dir=thirdparty_dir, - ) - package_envts_not_built, _wheel_filenames_built = results - - print("***ADD*** ABOUT AND LICENSES") - utils_thirdparty.add_fetch_or_update_about_and_license_files( - dest_dir=thirdparty_dir, - strip_classifiers=strip_classifiers, - ) - - # report issues - for name, version in src_name_ver_not_fetched: - print(f"{name}=={version}: Failed to fetch source distribution.") - - for package, envt in package_envts_not_built: - print( - f"{package.name}=={package.version}: Failed to build wheel " - f"on {envt.operating_system} for Python {envt.python_version}" - ) - - print("***FIND PROBLEMS***") - utils_thirdparty.find_problems(dest_dir=thirdparty_dir) - - -if __name__ == "__main__": - fix_thirdparty_dir() diff --git a/etc/scripts/gen_pypi_simple.py b/etc/scripts/gen_pypi_simple.py index 53db9b0..8de2b96 100644 --- a/etc/scripts/gen_pypi_simple.py +++ b/etc/scripts/gen_pypi_simple.py @@ -5,65 +5,25 @@ # Copyright (c) 2010 David Wolever . All rights reserved. # originally from https://github.com/wolever/pip2pi +import hashlib import os import re import shutil - +from collections import defaultdict from html import escape from pathlib import Path +from typing import NamedTuple """ -name: pip compatibility tags -version: 20.3.1 -download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/models/wheel.py -copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) -license_expression: mit -notes: the weel name regex is copied from pip-20.3.1 pip/_internal/models/wheel.py - -Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +Generate a PyPI simple index froma directory. """ -get_wheel_from_filename = re.compile( - r"""^(?P(?P.+?)-(?P.*?)) - ((-(?P\d[^-]*?))?-(?P.+?)-(?P.+?)-(?P.+?) - \.whl)$""", - re.VERBOSE, -).match - -sdist_exts = ( - ".tar.gz", - ".tar.bz2", - ".zip", - ".tar.xz", -) -wheel_ext = ".whl" -app_ext = ".pyz" -dist_exts = sdist_exts + (wheel_ext, app_ext) class InvalidDistributionFilename(Exception): pass -def get_package_name_from_filename(filename, normalize=True): +def get_package_name_from_filename(filename): """ Return the package name extracted from a package ``filename``. Optionally ``normalize`` the name according to distribution name rules. @@ -132,18 +92,99 @@ def get_package_name_from_filename(filename, normalize=True): if not name: raise InvalidDistributionFilename(filename) - if normalize: - name = name.lower().replace("_", "-") + name = normalize_name(name) return name -def build_pypi_index(directory, write_index=False): +def normalize_name(name): + """ + Return a normalized package name per PEP503, and copied from + https://www.python.org/dev/peps/pep-0503/#id4 + """ + return name and re.sub(r"[-_.]+", "-", name).lower() or name + + +def build_per_package_index(pkg_name, packages, base_url): + """ + Return an HTML document as string representing the index for a package + """ + document = [] + header = f""" + + + + Links for {pkg_name} + + """ + document.append(header) + + for package in packages: + document.append(package.simple_index_entry(base_url)) + + footer = """ + +""" + document.append(footer) + return "\n".join(document) + + +def build_links_package_index(packages_by_package_name, base_url): + """ + Return an HTML document as string which is a links index of all packages + """ + document = [] + header = f""" + + + Links for all packages + + """ + document.append(header) + + for _name, packages in packages_by_package_name.items(): + for package in packages: + document.append(package.simple_index_entry(base_url)) + + footer = """ + +""" + document.append(footer) + return "\n".join(document) + + +class Package(NamedTuple): + name: str + index_dir: Path + archive_file: Path + checksum: str + + @classmethod + def from_file(cls, name, index_dir, archive_file): + with open(archive_file, "rb") as f: + checksum = hashlib.sha256(f.read()).hexdigest() + return cls( + name=name, + index_dir=index_dir, + archive_file=archive_file, + checksum=checksum, + ) + + def simple_index_entry(self, base_url): + return ( + f' ' + f"{self.archive_file.name}
" + ) + + +def build_pypi_index(directory, base_url="https://thirdparty.aboutcode.org/pypi"): """ - Using a ``directory`` directory of wheels and sdists, create the a PyPI simple - directory index at ``directory``/simple/ populated with the proper PyPI simple - index directory structure crafted using symlinks. + Using a ``directory`` directory of wheels and sdists, create the a PyPI + simple directory index at ``directory``/simple/ populated with the proper + PyPI simple index directory structure crafted using symlinks. WARNING: The ``directory``/simple/ directory is removed if it exists. + NOTE: in addition to the a PyPI simple index.html there is also a links.html + index file generated which is suitable to use with pip's --find-links """ directory = Path(directory) @@ -153,14 +194,15 @@ def build_pypi_index(directory, write_index=False): shutil.rmtree(str(index_dir), ignore_errors=True) index_dir.mkdir(parents=True) + packages_by_package_name = defaultdict(list) - if write_index: - simple_html_index = [ - "PyPI Simple Index", - "", - ] + # generate the main simple index.html + simple_html_index = [ + "", + "PyPI Simple Index", + '' '', + ] - package_names = set() for pkg_file in directory.iterdir(): pkg_filename = pkg_file.name @@ -172,23 +214,99 @@ def build_pypi_index(directory, write_index=False): ): continue - pkg_name = get_package_name_from_filename(pkg_filename) + pkg_name = get_package_name_from_filename( + filename=pkg_filename, + ) pkg_index_dir = index_dir / pkg_name pkg_index_dir.mkdir(parents=True, exist_ok=True) pkg_indexed_file = pkg_index_dir / pkg_filename + link_target = Path("../..") / pkg_filename pkg_indexed_file.symlink_to(link_target) - if write_index and pkg_name not in package_names: + if pkg_name not in packages_by_package_name: esc_name = escape(pkg_name) simple_html_index.append(f'{esc_name}
') - package_names.add(pkg_name) - if write_index: - simple_html_index.append("") - index_html = index_dir / "index.html" - index_html.write_text("\n".join(simple_html_index)) + packages_by_package_name[pkg_name].append( + Package.from_file( + name=pkg_name, + index_dir=pkg_index_dir, + archive_file=pkg_file, + ) + ) + + # finalize main index + simple_html_index.append("") + index_html = index_dir / "index.html" + index_html.write_text("\n".join(simple_html_index)) + + # also generate the simple index.html of each package, listing all its versions. + for pkg_name, packages in packages_by_package_name.items(): + per_package_index = build_per_package_index( + pkg_name=pkg_name, + packages=packages, + base_url=base_url, + ) + pkg_index_dir = packages[0].index_dir + ppi_html = pkg_index_dir / "index.html" + ppi_html.write_text(per_package_index) + + # also generate the a links.html page with all packages. + package_links = build_links_package_index( + packages_by_package_name=packages_by_package_name, + base_url=base_url, + ) + links_html = index_dir / "links.html" + links_html.write_text(package_links) + + +""" +name: pip-wheel +version: 20.3.1 +download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/models/wheel.py +copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) +license_expression: mit +notes: the wheel name regex is copied from pip-20.3.1 pip/_internal/models/wheel.py + +Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +get_wheel_from_filename = re.compile( + r"""^(?P(?P.+?)-(?P.*?)) + ((-(?P\d[^-]*?))?-(?P.+?)-(?P.+?)-(?P.+?) + \.whl)$""", + re.VERBOSE, +).match + +sdist_exts = ( + ".tar.gz", + ".tar.bz2", + ".zip", + ".tar.xz", +) +wheel_ext = ".whl" +app_ext = ".pyz" +dist_exts = sdist_exts + (wheel_ext, app_ext) if __name__ == "__main__": import sys diff --git a/etc/scripts/publish_files.py b/etc/scripts/publish_files.py deleted file mode 100644 index 8669363..0000000 --- a/etc/scripts/publish_files.py +++ /dev/null @@ -1,208 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/scancode-toolkit for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import hashlib -import os -import sys - -from pathlib import Path - -import click -import requests -import utils_thirdparty - -from github_release_retry import github_release_retry as grr - -""" -Create GitHub releases and upload files there. -""" - - -def get_files(location): - """ - Return an iterable of (filename, Path, md5) tuples for files in the `location` - directory tree recursively. - """ - for top, _dirs, files in os.walk(location): - for filename in files: - pth = Path(os.path.join(top, filename)) - with open(pth, "rb") as fi: - md5 = hashlib.md5(fi.read()).hexdigest() - yield filename, pth, md5 - - -def get_etag_md5(url): - """ - Return the cleaned etag of URL `url` or None. - """ - headers = utils_thirdparty.get_remote_headers(url) - headers = {k.lower(): v for k, v in headers.items()} - etag = headers.get("etag") - if etag: - etag = etag.strip('"').lower() - return etag - - -def create_or_update_release_and_upload_directory( - user, - repo, - tag_name, - token, - directory, - retry_limit=10, - description=None, -): - """ - Create or update a GitHub release at https://github.com// for - `tag_name` tag using the optional `description` for this release. - Use the provided `token` as a GitHub token for API calls authentication. - Upload all files found in the `directory` tree to that GitHub release. - Retry API calls up to `retry_limit` time to work around instability the - GitHub API. - - Remote files that are not the same as the local files are deleted and re- - uploaded. - """ - release_homepage_url = f"https://github.com/{user}/{repo}/releases/{tag_name}" - - # scrape release page HTML for links - urls_by_filename = { - os.path.basename(l): l - for l in utils_thirdparty.get_paths_or_urls(links_url=release_homepage_url) - } - - # compute what is new, modified or unchanged - print(f"Compute which files is new, modified or unchanged in {release_homepage_url}") - - new_to_upload = [] - unchanged_to_skip = [] - modified_to_delete_and_reupload = [] - for filename, pth, md5 in get_files(directory): - url = urls_by_filename.get(filename) - if not url: - print(f"{filename} content is NEW, will upload") - new_to_upload.append(pth) - continue - - out_of_date = get_etag_md5(url) != md5 - if out_of_date: - print(f"{url} content is CHANGED based on md5 etag, will re-upload") - modified_to_delete_and_reupload.append(pth) - else: - # print(f'{url} content is IDENTICAL, skipping upload based on Etag') - unchanged_to_skip.append(pth) - print(".") - - ghapi = grr.GithubApi( - github_api_url="https://api.github.com", - user=user, - repo=repo, - token=token, - retry_limit=retry_limit, - ) - - # yank modified - print( - f"Unpublishing {len(modified_to_delete_and_reupload)} published but " - f"locally modified files in {release_homepage_url}" - ) - - release = ghapi.get_release_by_tag(tag_name) - - for pth in modified_to_delete_and_reupload: - filename = os.path.basename(pth) - asset_id = ghapi.find_asset_id_by_file_name(filename, release) - print(f" Unpublishing file: {filename}).") - response = ghapi.delete_asset(asset_id) - if response.status_code != requests.codes.no_content: # NOQA - raise Exception(f"failed asset deletion: {response}") - - # finally upload new and modified - to_upload = new_to_upload + modified_to_delete_and_reupload - print(f"Publishing with {len(to_upload)} files to {release_homepage_url}") - release = grr.Release(tag_name=tag_name, body=description) - grr.make_release(ghapi, release, to_upload) - - -TOKEN_HELP = ( - "The Github personal acess token is used to authenticate API calls. " - "Required unless you set the GITHUB_TOKEN environment variable as an alternative. " - "See for details: https://github.com/settings/tokens and " - "https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token" -) - - -@click.command() -@click.option( - "--user-repo-tag", - help="The GitHub qualified repository user/name/tag in which " - "to create the release such as in nexB/thirdparty/pypi", - type=str, - required=True, -) -@click.option( - "-d", - "--directory", - help="The directory that contains files to upload to the release.", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False, resolve_path=True), - required=True, -) -@click.option( - "--token", - help=TOKEN_HELP, - default=os.environ.get("GITHUB_TOKEN", None), - type=str, - required=False, -) -@click.option( - "--description", - help="Text description for the release. Ignored if the release exists.", - default=None, - type=str, - required=False, -) -@click.option( - "--retry_limit", - help="Number of retries when making failing GitHub API calls. " - "Retrying helps work around transient failures of the GitHub API.", - type=int, - default=10, -) -@click.help_option("-h", "--help") -def publish_files( - user_repo_tag, - directory, - retry_limit=10, - token=None, - description=None, -): - """ - Publish all the files in DIRECTORY as assets to a GitHub release. - Either create or update/replace remote files' - """ - if not token: - click.secho("--token required option is missing.") - click.secho(TOKEN_HELP) - sys.exit(1) - - user, repo, tag_name = user_repo_tag.split("/") - - create_or_update_release_and_upload_directory( - user=user, - repo=repo, - tag_name=tag_name, - description=description, - retry_limit=retry_limit, - token=token, - directory=directory, - ) - - -if __name__ == "__main__": - publish_files() diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index 7753ea0..fbc456d 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -14,95 +14,63 @@ """ Utilities to manage requirements files and call pip. NOTE: this should use ONLY the standard library and not import anything else -becasue this is used for boostrapping. +because this is used for boostrapping with no requirements installed. """ -def load_requirements(requirements_file="requirements.txt", force_pinned=True): +def load_requirements(requirements_file="requirements.txt", with_unpinned=False): """ Yield package (name, version) tuples for each requirement in a `requirement` - file. Every requirement versions must be pinned if `force_pinned` is True. - Otherwise un-pinned requirements are returned with a None version + file. Only accept requirements pinned to an exact version. """ with open(requirements_file) as reqs: req_lines = reqs.read().splitlines(False) - return get_required_name_versions(req_lines, force_pinned) + return get_required_name_versions(req_lines, with_unpinned=with_unpinned) -def get_required_name_versions( - requirement_lines, - force_pinned=True, -): +def get_required_name_versions(requirement_lines, with_unpinned=False): """ Yield required (name, version) tuples given a`requirement_lines` iterable of - requirement text lines. Every requirement versions must be pinned if - `force_pinned` is True. Otherwise un-pinned requirements are returned with a - None version. - + requirement text lines. Only accept requirements pinned to an exact version. """ + for req_line in requirement_lines: req_line = req_line.strip() if not req_line or req_line.startswith("#"): continue - if force_pinned: - if "==" not in req_line: - raise Exception(f"Requirement version is not pinned: {req_line}") - name = req_line - version = None - else: - if req_line.startswith("-"): - print(f"Requirement skipped, is not supported: {req_line}") - - if "==" in req_line: - name, _, version = req_line.partition("==") - version = version.lower().strip() - else: - # FIXME: we do not support unpinned requirements yet! - name = strip_reqs(req_line) - version = None - - name = name.lower().strip() - yield name, version - - -def strip_reqs(line): - """ - Return a name given a pip reuirement text ``line` striping version and - requirements. - - For example:: - - >>> s = strip_reqs("foo <=12, >=13,!=12.6") - >>> assert s == "foo" - """ - if "--" in line: - raise Exception(f"Unsupported requirement style: {line}") - - line = line.strip() - - ops = ">>> assert get_name_version("foo==1.2.3") == ("foo", "1.2.3") + >>> assert get_name_version("fooA==1.2.3.DEV1") == ("fooa", "1.2.3.dev1") + >>> assert get_name_version("foo==1.2.3", with_unpinned=False) == ("foo", "1.2.3") + >>> assert get_name_version("foo", with_unpinned=True) == ("foo", "") + >>> assert get_name_version("foo>=1.2", with_unpinned=True) == ("foo", ""), get_name_version("foo>=1.2") + >>> try: + ... assert not get_name_version("foo", with_unpinned=False) + ... except Exception as e: + ... assert "Requirement version must be pinned" in str(e) """ - requires = [c for c in requires.splitlines(False) if c] - if not requires: - return [] - - requires = ["".join(r.split()) for r in requires if r and r.strip()] - return sorted(requires) + requirement = requirement and "".join(requirement.lower().split()) + assert requirement, f"specifier is required is empty:{requirement!r}" + name, operator, version = split_req(requirement) + assert name, f"Name is required: {requirement}" + is_pinned = operator == "==" + if with_unpinned: + version = "" + else: + assert is_pinned and version, f"Requirement version must be pinned: {requirement}" + return name, version def lock_requirements(requirements_file="requirements.txt", site_packages_dir=None): @@ -139,8 +107,47 @@ def lock_dev_requirements( def get_installed_reqs(site_packages_dir): """ - Return the installed pip requirements as text found in `site_packages_dir` as a text. + Return the installed pip requirements as text found in `site_packages_dir` + as a text. """ - # Also include these packages in the output with --all: wheel, distribute, setuptools, pip + # Also include these packages in the output with --all: wheel, distribute, + # setuptools, pip args = ["pip", "freeze", "--exclude-editable", "--all", "--path", site_packages_dir] return subprocess.check_output(args, encoding="utf-8") + + +comparators = ( + "===", + "~=", + "!=", + "==", + "<=", + ">=", + ">", + "<", +) + +_comparators_re = r"|".join(comparators) +version_splitter = re.compile(rf"({_comparators_re})") + + +def split_req(req): + """ + Return a three-tuple of (name, comparator, version) given a ``req`` + requirement specifier string. Each segment may be empty. Spaces are removed. + + For example: + >>> assert split_req("foo==1.2.3") == ("foo", "==", "1.2.3"), split_req("foo==1.2.3") + >>> assert split_req("foo") == ("foo", "", ""), split_req("foo") + >>> assert split_req("==1.2.3") == ("", "==", "1.2.3"), split_req("==1.2.3") + >>> assert split_req("foo >= 1.2.3 ") == ("foo", ">=", "1.2.3"), split_req("foo >= 1.2.3 ") + >>> assert split_req("foo>=1.2") == ("foo", ">=", "1.2"), split_req("foo>=1.2") + """ + assert req + # do not allow multiple constraints and tags + assert not any(c in req for c in ",;") + req = "".join(req.split()) + if not any(c in req for c in comparators): + return req, "", "" + segments = version_splitter.split(req, maxsplit=1) + return tuple(segments) diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index a2fbe4e..e303053 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -11,12 +11,10 @@ from collections import defaultdict import email import itertools -import operator import os import re import shutil import subprocess -import tarfile import tempfile import time import urllib @@ -26,29 +24,30 @@ import packageurl import requests import saneyaml -import utils_pip_compatibility_tags -import utils_pypi_supported_tags - from commoncode import fileutils from commoncode.hash import multi_checksums from commoncode.text import python_safe_name from packaging import tags as packaging_tags from packaging import version as packaging_version +from urllib.parse import quote_plus + +import utils_pip_compatibility_tags from utils_requirements import load_requirements """ Utilities to manage Python thirparty libraries source, binaries and metadata in local directories and remote repositories. -- update pip requirement files from installed packages for prod. and dev. -- build and save wheels for all required packages -- also build variants for wheels with native code for all each supported - operating systems (Linux, macOS, Windows) and Python versions (3.x) - combinations using remote Ci jobs -- collect source distributions for all required packages -- keep in sync wheels, distributions, ABOUT and LICENSE files to a PyPI-like - repository (using GitHub) -- create, update and fetch ABOUT, NOTICE and LICENSE metadata for all distributions +- download wheels for packages for all each supported operating systems + (Linux, macOS, Windows) and Python versions (3.x) combinations + +- download sources for packages (aka. sdist) + +- create, update and download ABOUT, NOTICE and LICENSE metadata for these + wheels and source distributions + +- update pip requirement files based on actually installed packages for + production and development Approach @@ -56,35 +55,65 @@ The processing is organized around these key objects: -- A PyPiPackage represents a PyPI package with its name and version. It tracks - the downloadable Distribution objects for that version: +- A PyPiPackage represents a PyPI package with its name and version and the + metadata used to populate an .ABOUT file and document origin and license. + It contains the downloadable Distribution objects for that version: - - one Sdist source Distribution object - - a list of Wheel binary Distribution objects + - one Sdist source Distribution + - a list of Wheel binary Distribution - A Distribution (either a Wheel or Sdist) is identified by and created from its - filename. It also has the metadata used to populate an .ABOUT file and - document origin and license. A Distribution can be fetched from Repository. - Metadata can be loaded from and dumped to ABOUT files and optionally from - DejaCode package data. + filename as well as its name and version. + A Distribution is fetched from a Repository. + Distribution metadata can be loaded from and dumped to ABOUT files. + +- A Wheel binary Distribution can have Python/Platform/OS tags it supports and + was built for and these tags can be matched to an Environment. + +- An Environment is a combination of a Python version and operating system + (e.g., platfiorm and ABI tags.) and is represented by the "tags" it supports. + +- A plain LinksRepository which is just a collection of URLs scrape from a web + page such as HTTP diretory listing. It is used either with pip "--find-links" + option or to fetch ABOUT and LICENSE files. + +- A PypiSimpleRepository is a PyPI "simple" index where a HTML page is listing + package name links. Each such link points to an HTML page listing URLs to all + wheels and sdsist of all versions of this package. + +PypiSimpleRepository and Packages are related through packages name, version and +filenames. + +The Wheel models code is partially derived from the mit-licensed pip and the +Distribution/Wheel/Sdist design has been heavily inspired by the packaging- +dists library https://github.com/uranusjr/packaging-dists by Tzu-ping Chung +""" + +""" +Wheel downloader -- An Environment is a combination of a Python version and operating system. - A Wheel Distribution also has Python/OS tags is supports and these can be - supported in a given Environment. +- parse requirement file +- create a TODO queue of requirements to process +- done: create an empty map of processed binary requirements as {package name: (list of versions/tags} -- Paths or URLs to "filenames" live in a Repository, either a plain - LinksRepository (an HTML page listing URLs or a local directory) or a - PypiRepository (a PyPI simple index where each package name has an HTML page - listing URLs to all distribution types and versions). - Repositories and Distributions are related through filenames. + +- while we have package reqs in TODO queue, process one requirement: + - for each PyPI simple index: + - fetch through cache the PyPI simple index for this package + - for each environment: + - find a wheel matching pinned requirement in this index + - if file exist locally, continue + - fetch the wheel for env + - IF pure, break, no more needed for env + - collect requirement deps from wheel metadata and add to queue + - if fetched, break, otherwise display error message - The Wheel models code is partially derived from the mit-licensed pip and the - Distribution/Wheel/Sdist design has been heavily inspired by the packaging- - dists library https://github.com/uranusjr/packaging-dists by Tzu-ping Chung """ -TRACE = False +TRACE = True +TRACE_DEEP = False +TRACE_ULTRA_DEEP = False # Supported environments PYTHON_VERSIONS = "36", "37", "38", "39", "310" @@ -106,16 +135,11 @@ def get_python_dot_version(version): ABIS_BY_PYTHON_VERSION = { - "36": ["cp36", "cp36m"], - "37": ["cp37", "cp37m"], - "38": ["cp38", "cp38m"], - "39": ["cp39", "cp39m"], - "310": ["cp310", "cp310m"], - "36": ["cp36", "abi3"], - "37": ["cp37", "abi3"], - "38": ["cp38", "abi3"], - "39": ["cp39", "abi3"], - "310": ["cp310", "abi3"], + "36": ["cp36", "cp36m", "abi3"], + "37": ["cp37", "cp37m", "abi3"], + "38": ["cp38", "cp38m", "abi3"], + "39": ["cp39", "cp39m", "abi3"], + "310": ["cp310", "cp310m", "abi3"], } PLATFORMS_BY_OS = { @@ -154,7 +178,13 @@ def get_python_dot_version(version): THIRDPARTY_DIR = "thirdparty" CACHE_THIRDPARTY_DIR = ".cache/thirdparty" -REMOTE_LINKS_URL = "https://thirdparty.aboutcode.org/pypi" +ABOUT_BASE_URL = "https://thirdparty.aboutcode.org/pypi" + +ABOUT_PYPI_SIMPLE_URL = f"{ABOUT_BASE_URL}/simple" +ABOUT_LINKS_URL = f"{ABOUT_PYPI_SIMPLE_URL}/links.html" + +PYPI_SIMPLE_URL = "https://pypi.org/simple" +PYPI_INDEXES = (PYPI_SIMPLE_URL, ABOUT_PYPI_SIMPLE_URL) EXTENSIONS_APP = (".pyz",) EXTENSIONS_SDIST = ( @@ -171,170 +201,134 @@ def get_python_dot_version(version): ) EXTENSIONS = EXTENSIONS_INSTALLABLE + EXTENSIONS_ABOUT + EXTENSIONS_APP -PYPI_SIMPLE_URL = "https://pypi.org/simple" - LICENSEDB_API_URL = "https://scancode-licensedb.aboutcode.org" LICENSING = license_expression.Licensing() -# time to wait build for in seconds, as a string -# 0 measn no wait -DEFAULT_ROMP_BUILD_WAIT = "5" +collect_urls = re.compile('href="([^"]+)"').findall ################################################################################ -# -# Fetch remote wheels and sources locally -# +# Fetch wheels and sources locally ################################################################################ -def fetch_wheels( - environment=None, - requirements_file="requirements.txt", - allow_unpinned=False, +class DistributionNotFound(Exception): + pass + + +def download_wheel( + name, + version, + environment, dest_dir=THIRDPARTY_DIR, - remote_links_url=REMOTE_LINKS_URL, + index_urls=PYPI_INDEXES, ): """ - Download all of the wheel of packages listed in the ``requirements_file`` - requirements file into ``dest_dir`` directory. + Download the wheels binary distribution(s) of package ``name`` and + ``version`` matching the ``environment`` Environment constraints from the + PyPI simple repository ``index_urls`` list of URLs into the ``dest_dir`` + directory. - Only get wheels for the ``environment`` Enviromnent constraints. If the - provided ``environment`` is None then the current Python interpreter - environment is used implicitly. + Raise a DistributionNotFound if no wheel is not found. Otherwise, return a + tuple of lists of (fetched_wheel_filenames, existing_wheel_filenames) + """ + if TRACE_DEEP: + print(f" download_wheel: {name}=={version}: {environment}") - Only accept pinned requirements (e.g. with a version) unless - ``allow_unpinned`` is True. + fetched_wheel_filenames = [] + existing_wheel_filenames = [] + try: + for pypi_package in get_package_versions( + name=name, + version=version, + index_urls=index_urls, + ): + if not pypi_package.wheels: + continue - Use exclusively direct downloads from a remote repo at URL - ``remote_links_url``. If ``remote_links_url`` is a path, use this as a - directory of links instead of a URL. + supported_wheels = list(pypi_package.get_supported_wheels(environment=environment)) + if not supported_wheels: + continue - Yield tuples of (PypiPackage, error) where is None on success. - """ - missed = [] + for wheel in supported_wheels: + if os.path.exists(os.path.join(dest_dir, wheel.filename)): + # do not refetch + existing_wheel_filenames.append(wheel.filename) + continue - if not allow_unpinned: - force_pinned = True - else: - force_pinned = False + if TRACE: + print(f" Fetching wheel from index: {wheel.download_url}") + fetched_wheel_filename = wheel.download(dest_dir=dest_dir) + fetched_wheel_filenames.add(fetched_wheel_filename) - try: - rrp = list( - get_required_remote_packages( - requirements_file=requirements_file, - force_pinned=force_pinned, - remote_links_url=remote_links_url, - ) - ) except Exception as e: - raise Exception( - dict( - requirements_file=requirements_file, - force_pinned=force_pinned, - remote_links_url=remote_links_url, - ) - ) from e - - fetched_filenames = set() - for name, version, package in rrp: - if not package: - missed.append( - ( - name, - version, - ) - ) - nv = f"{name}=={version}" if version else name - yield None, f"fetch_wheels: Missing package in remote repo: {nv}" + raise DistributionNotFound(f"Failed to fetch wheel: {name}=={version}: {e}") from e - else: - fetched_filename = package.fetch_wheel( - environment=environment, - fetched_filenames=fetched_filenames, - dest_dir=dest_dir, - ) + if not fetched_wheel_filenames and not existing_wheel_filenames: + raise DistributionNotFound(f"Failed to fetch wheel: {name}=={version}: No wheel found") - if fetched_filename: - fetched_filenames.add(fetched_filename) - error = None - else: - if fetched_filename in fetched_filenames: - error = None - else: - error = f"Failed to fetch" - yield package, error - - if missed: - rr = get_remote_repo() - print() - print(f"===> fetch_wheels: Missed some packages") - for n, v in missed: - nv = f"{n}=={v}" if v else n - print(f"Missed package {nv} in remote repo, has only:") - for pv in rr.get_versions(n): - print(" ", pv) - raise Exception("Missed some packages in remote repo") + return fetched_wheel_filenames, existing_wheel_filenames -def fetch_sources( - requirements_file="requirements.txt", - allow_unpinned=False, +def download_sdist( + name, + version, dest_dir=THIRDPARTY_DIR, - remote_links_url=REMOTE_LINKS_URL, + index_urls=PYPI_INDEXES, ): """ - Download all of the dependent package sources listed in the - ``requirements_file`` requirements file into ``dest_dir`` destination - directory. - - Use direct downloads to achieve this (not pip download). Use exclusively the - packages found from a remote repo at URL ``remote_links_url``. If - ``remote_links_url`` is a path, use this as a directory of links instead of - a URL. + Download the sdist source distribution of package ``name`` and ``version`` + from the PyPI simple repository ``index_urls`` list of URLs into the + ``dest_dir`` directory. - Only accept pinned requirements (e.g. with a version) unless - ``allow_unpinned`` is True. - - Yield tuples of (PypiPackage, error message) for each package where error - message will empty on success. + Raise a DistributionNotFound if this was not found. Return the filename if + downloaded and False if not downloaded because it already exists. """ - missed = [] + if TRACE_DEEP: + print(f"download_sdist: {name}=={version}: ") - if not allow_unpinned: - force_pinned = True - else: - force_pinned = False + try: + for pypi_package in get_package_versions( + name=name, + version=version, + index_urls=index_urls, + ): + if not pypi_package.sdist: + continue - rrp = list( - get_required_remote_packages( - requirements_file=requirements_file, - force_pinned=force_pinned, - remote_links_url=remote_links_url, - ) - ) + if os.path.exists(os.path.join(dest_dir, pypi_package.sdist.filename)): + # do not refetch + return False + if TRACE: + print(f" Fetching sources from index: {pypi_package.sdist.download_url}") + fetched = pypi_package.sdist.download(dest_dir=dest_dir) + if fetched: + return pypi_package.sdist.filename - for name, version, package in rrp: - if not package: - missed.append( - ( - name, - name, - ) - ) - nv = f"{name}=={version}" if version else name - yield None, f"fetch_sources: Missing package in remote repo: {nv}" + except Exception as e: + raise DistributionNotFound(f"Failed to fetch sdist: {name}=={version}: {e}") from e - elif not package.sdist: - yield package, f"Missing sdist in links" + raise DistributionNotFound(f"Failed to fetch sdist: {name}=={version}: No sources found") - else: - fetched = package.fetch_sdist(dest_dir=dest_dir) - error = f"Failed to fetch" if not fetched else None - yield package, error - if missed: - raise Exception(f"Missing source packages in {remote_links_url}", missed) +def get_package_versions( + name, + version=None, + index_urls=PYPI_INDEXES, +): + """ + Yield PypiPackages with ``name`` and ``version`` from the PyPI simple + repository ``index_urls`` list of URLs. + If ``version`` is not provided, return the latest available versions. + """ + for index_url in index_urls: + try: + repo = get_pypi_repo(index_url) + package = repo.get_package(name, version) + if package: + yield package + except RemoteNotFetchedException as e: + print(f"Failed to fetch PyPI package {name} @ {version} info from {index_url}: {e}") ################################################################################ # @@ -387,7 +381,7 @@ def sortable_name_version(self): @classmethod def sorted(cls, namevers): - return sorted(namevers, key=cls.sortable_name_version) + return sorted(namevers or [], key=cls.sortable_name_version) @attr.attributes @@ -411,13 +405,6 @@ class Distribution(NameVer): metadata=dict(help="File name."), ) - path_or_url = attr.ib( - repr=False, - type=str, - default="", - metadata=dict(help="Path or download URL."), - ) - sha256 = attr.ib( repr=False, type=str, @@ -546,21 +533,60 @@ def package_url(self): @property def download_url(self): - if self.path_or_url and self.path_or_url.startswith("https://"): - return self.path_or_url - else: - return self.get_best_download_url() + return self.get_best_download_url() + + def get_best_download_url( + self, + index_urls=tuple([PYPI_SIMPLE_URL, ABOUT_PYPI_SIMPLE_URL]), + ): + """ + Return the best download URL for this distribution where best means that + PyPI is better and our selfhosted repo URLs are second. + If none is found, return a synthetic remote URL. + """ + for index_url in index_urls: + pypi_package = get_pypi_package( + name=self.normalized_name, + version=self.version, + index_url=index_url, + ) + if pypi_package: + if isinstance(pypi_package, tuple): + raise Exception("############", repr(pypi_package)) + try: + pypi_url = pypi_package.get_url_for_filename(self.filename) + except Exception as e: + raise Exception(repr(pypi_package)) from e + if pypi_url: + return pypi_url + + def download(self, dest_dir=THIRDPARTY_DIR): + """ + Download this distribution into `dest_dir` directory. + Return the fetched filename. + """ + assert self.filename + if TRACE: + print( + f"Fetching distribution of {self.name}=={self.version}:", + self.filename, + ) + + fetch_and_save_path_or_url( + filename=self.filename, + dest_dir=dest_dir, + path_or_url=self.path_or_url, + as_text=False, + ) + return self.filename @property def about_filename(self): return f"{self.filename}.ABOUT" - def has_about_file(self, dest_dir=THIRDPARTY_DIR): - return os.path.exists(os.path.join(dest_dir, self.about_filename)) - @property def about_download_url(self): - return self.build_remote_download_url(self.about_filename) + return f"{ABOUT_BASE_URL}/{self.about_filename}" @property def notice_filename(self): @@ -568,7 +594,7 @@ def notice_filename(self): @property def notice_download_url(self): - return self.build_remote_download_url(self.notice_filename) + return f"{ABOUT_BASE_URL}/{self.notice_filename}" @classmethod def from_path_or_url(cls, path_or_url): @@ -601,81 +627,10 @@ def from_filename(cls, filename): Return a distribution built from the data found in a `filename` string. Raise an exception if this is not a valid filename """ + filename = os.path.basename(filename.strip("/")) clazz = cls.get_dist_class(filename) return clazz.from_filename(filename) - @classmethod - def from_data(cls, data, keep_extra=False): - """ - Return a distribution built from a `data` mapping. - """ - filename = data["filename"] - dist = cls.from_filename(filename) - dist.update(data, keep_extra=keep_extra) - return dist - - @classmethod - def from_dist(cls, data, dist): - """ - Return a distribution built from a `data` mapping and update it with data - from another dist Distribution. Return None if it cannot be created - """ - # We can only create from a dist of the same package - has_same_key_fields = all( - data.get(kf) == getattr(dist, kf, None) for kf in ("type", "namespace", "name") - ) - if not has_same_key_fields: - print( - f"Missing key fields: Cannot derive a new dist from data: {data} and dist: {dist}" - ) - return - - has_key_field_values = all(data.get(kf) for kf in ("type", "name", "version")) - if not has_key_field_values: - print( - f"Missing key field values: Cannot derive a new dist from data: {data} and dist: {dist}" - ) - return - - data = dict(data) - # do not overwrite the data with the other dist - # only supplement - data.update({k: v for k, v in dist.get_updatable_data().items() if not data.get(k)}) - return cls.from_data(data) - - @classmethod - def build_remote_download_url(cls, filename, base_url=REMOTE_LINKS_URL): - """ - Return a direct download URL for a file in our remote repo - """ - return f"{base_url}/{filename}" - - def get_best_download_url(self): - """ - Return the best download URL for this distribution where best means that - PyPI is better and our own remote repo URLs are second. - If none is found, return a synthetic remote URL. - """ - name = self.normalized_name - version = self.version - filename = self.filename - - pypi_package = get_pypi_package(name=name, version=version) - if pypi_package: - pypi_url = pypi_package.get_url_for_filename(filename) - if pypi_url: - return pypi_url - - remote_package = get_remote_package(name=name, version=version) - if remote_package: - remote_url = remote_package.get_url_for_filename(filename) - if remote_url: - return remote_url - else: - # the package may not have been published yet, so we craft a URL - # using our remote base URL - return self.build_remote_download_url(self.filename) - def purl_identifiers(self, skinny=False): """ Return a mapping of non-empty identifier name/values for the purl @@ -781,9 +736,11 @@ def save_if_modified(location, content): fo.write(content) return True + as_about = self.to_about() + save_if_modified( location=os.path.join(dest_dir, self.about_filename), - content=saneyaml.dump(self.to_about()), + content=saneyaml.dump(as_about), ) notice_text = self.notice_text and self.notice_text.strip() @@ -844,7 +801,10 @@ def load_remote_about_data(self): NOTICE file if any. Return True if the data was updated. """ try: - about_text = fetch_content_from_path_or_url_through_cache(self.about_download_url) + about_text = fetch_content_from_path_or_url_through_cache( + path_or_url=self.about_download_url, + as_text=True, + ) except RemoteNotFetchedException: return False @@ -855,7 +815,10 @@ def load_remote_about_data(self): notice_file = about_data.pop("notice_file", None) if notice_file: try: - notice_text = fetch_content_from_path_or_url_through_cache(self.notice_download_url) + notice_text = fetch_content_from_path_or_url_through_cache( + path_or_url=self.notice_download_url, + as_text=True, + ) if notice_text: about_data["notice_text"] = notice_text except RemoteNotFetchedException: @@ -892,26 +855,23 @@ def validate_checksums(self, dest_dir=THIRDPARTY_DIR): return False return True - def get_pip_hash(self): - """ - Return a pip hash option string as used in requirements for this dist. - """ - assert self.sha256, f"Missinh SHA256 for dist {self}" - return f"--hash=sha256:{self.sha256}" - def get_license_keys(self): try: - keys = LICENSING.license_keys(self.license_expression, unique=True, simple=True) + keys = LICENSING.license_keys( + self.license_expression, + unique=True, + simple=True, + ) except license_expression.ExpressionParseError: return ["unknown"] return keys def fetch_license_files(self, dest_dir=THIRDPARTY_DIR): """ - Fetch license files is missing in `dest_dir`. + Fetch license files if missing in `dest_dir`. Return True if license files were fetched. """ - paths_or_urls = get_remote_repo().links + urls = LinksRepository.from_url().links errors = [] extra_lic_names = [l.get("file") for l in self.extra_data.get("licenses", {})] extra_lic_names += [self.extra_data.get("license_file")] @@ -924,7 +884,7 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR): try: # try remotely first - lic_url = get_link_for_filename(filename=filename, paths_or_urls=paths_or_urls) + lic_url = get_license_link_for_filename(filename=filename, urls=urls) fetch_and_save_path_or_url( filename=filename, @@ -960,9 +920,17 @@ def extract_pkginfo(self, dest_dir=THIRDPARTY_DIR): Return the text of the first PKG-INFO or METADATA file found in the archive of this Distribution in `dest_dir`. Return None if not found. """ - fmt = "zip" if self.filename.endswith(".whl") else None - dist = os.path.join(dest_dir, self.filename) - with tempfile.TemporaryDirectory(prefix="pypi-tmp-extract") as td: + + fn = self.filename + if fn.endswith(".whl"): + fmt = "zip" + elif fn.endswith(".tar.gz"): + fmt = "gztar" + else: + fmt = None + + dist = os.path.join(dest_dir, fn) + with tempfile.TemporaryDirectory(prefix=f"pypi-tmp-extract-{fn}") as td: shutil.unpack_archive(filename=dist, extract_dir=td, format=fmt) # NOTE: we only care about the first one found in the dist # which may not be 100% right @@ -983,7 +951,7 @@ def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR): """ pkginfo_text = self.extract_pkginfo(dest_dir=dest_dir) if not pkginfo_text: - print(f"!!!!PKG-INFO not found in {self.filename}") + print(f"!!!!PKG-INFO/METADATA not found in {self.filename}") return raw_data = email.message_from_string(pkginfo_text) @@ -1075,6 +1043,20 @@ def update(self, data, overwrite=False, keep_extra=True): return updated +def get_license_link_for_filename(filename, urls): + """ + Return a link for `filename` found in the `links` list of URLs or paths. Raise an + exception if no link is found or if there are more than one link for that + file name. + """ + path_or_url = [l for l in urls if l.endswith(f"/{filename}")] + if not path_or_url: + raise Exception(f"Missing link to file: {filename}") + if not len(path_or_url) == 1: + raise Exception(f"Multiple links to file: {filename}: \n" + "\n".join(path_or_url)) + return path_or_url[0] + + class InvalidDistributionFilename(Exception): pass @@ -1243,15 +1225,12 @@ def is_supported_by_tags(self, tags): """ Return True is this wheel is compatible with one of a list of PEP 425 tags. """ + if TRACE_DEEP: + print() + print("is_supported_by_tags: tags:", tags) + print("self.tags:", self.tags) return not self.tags.isdisjoint(tags) - def is_supported_by_environment(self, environment): - """ - Return True if this wheel is compatible with the Environment - `environment`. - """ - return not self.is_supported_by_tags(environment.tags) - def to_filename(self): """ Return a wheel filename reconstructed from its fields (that may not be @@ -1306,8 +1285,8 @@ class PypiPackage(NameVer): sdist = attr.ib( repr=False, - type=str, - default="", + type=Sdist, + default=None, metadata=dict(help="Sdist source distribution for this package."), ) @@ -1328,22 +1307,14 @@ def specifier(self): else: return self.name - @property - def specifier_with_hashes(self): - """ - Return a requirement specifier for this package with --hash options for - all its distributions - """ - items = [self.specifier] - items += [d.get_pip_hashes() for d in self.get_distributions()] - return " \\\n ".join(items) - - def get_supported_wheels(self, environment): + def get_supported_wheels(self, environment, verbose=TRACE_ULTRA_DEEP): """ Yield all the Wheel of this package supported and compatible with the Environment `environment`. """ envt_tags = environment.tags() + if verbose: + print("get_supported_wheels: envt_tags:", envt_tags) for wheel in self.wheels: if wheel.is_supported_by_tags(envt_tags): yield wheel @@ -1369,6 +1340,8 @@ def package_from_dists(cls, dists): >>> assert package.wheels == [w1, w2] """ dists = list(dists) + if TRACE_DEEP: + print(f"package_from_dists: {dists}") if not dists: return @@ -1379,13 +1352,21 @@ def package_from_dists(cls, dists): package = PypiPackage(name=normalized_name, version=version) for dist in dists: - if dist.normalized_name != normalized_name or dist.version != version: + if dist.normalized_name != normalized_name: if TRACE: print( - f" Skipping inconsistent dist name and version: {dist} " - f'Expected instead package name: {normalized_name} and version: "{version}"' + f" Skipping inconsistent dist name: expected {normalized_name} got {dist}" ) continue + elif dist.version != version: + dv = packaging_version.parse(dist.version) + v = packaging_version.parse(version) + if dv != v: + if TRACE: + print( + f" Skipping inconsistent dist version: expected {version} got {dist}" + ) + continue if isinstance(dist, Sdist): package.sdist = dist @@ -1396,39 +1377,41 @@ def package_from_dists(cls, dists): else: raise Exception(f"Unknown distribution type: {dist}") + if TRACE_DEEP: + print(f"package_from_dists: {package}") + return package @classmethod - def packages_from_one_path_or_url(cls, path_or_url): + def packages_from_dir(cls, directory): """ - Yield PypiPackages built from files found in at directory path or the - URL to an HTML page (that will be fetched). + Yield PypiPackages built from files found in at directory path. """ - extracted_paths_or_urls = get_paths_or_urls(path_or_url) - return cls.packages_from_many_paths_or_urls(extracted_paths_or_urls) + base = os.path.abspath(directory) + paths = [os.path.join(base, f) for f in os.listdir(base) if f.endswith(EXTENSIONS)] + if TRACE_ULTRA_DEEP: + print("packages_from_dir: paths:", paths) + return cls.packages_from_many_paths_or_urls(paths) @classmethod def packages_from_many_paths_or_urls(cls, paths_or_urls): """ Yield PypiPackages built from a list of paths or URLs. """ - dists = cls.get_dists(paths_or_urls) + dists = cls.dists_from_paths_or_urls(paths_or_urls) + if TRACE_ULTRA_DEEP: + print("packages_from_many_paths_or_urls: dists:", dists) + dists = NameVer.sorted(dists) for _projver, dists_of_package in itertools.groupby( dists, key=NameVer.sortable_name_version, ): - yield PypiPackage.package_from_dists(dists_of_package) - - @classmethod - def get_versions_from_path_or_url(cls, name, path_or_url): - """ - Return a subset list from a list of PypiPackages version at `path_or_url` - that match PypiPackage `name`. - """ - packages = cls.packages_from_one_path_or_url(path_or_url) - return cls.get_versions(name, packages) + package = PypiPackage.package_from_dists(dists_of_package) + if TRACE_ULTRA_DEEP: + print("packages_from_many_paths_or_urls", package) + yield package @classmethod def get_versions(cls, name, packages): @@ -1451,15 +1434,6 @@ def get_latest_version(cls, name, packages): return return versions[-1] - @classmethod - def get_outdated_versions(cls, name, packages): - """ - Return all versions except the latest version of PypiPackage `name` from a - list of `packages`. - """ - versions = cls.get_versions(name, packages) - return versions[:-1] - @classmethod def get_name_version(cls, name, version, packages): """ @@ -1467,100 +1441,23 @@ def get_name_version(cls, name, version, packages): or None if it is not found. If `version` is None, return the latest version found. """ - if version is None: + if TRACE_ULTRA_DEEP: + print("get_name_version:", name, version, packages) + if not version: return cls.get_latest_version(name, packages) nvs = [p for p in cls.get_versions(name, packages) if p.version == version] if not nvs: - return + return name, version if len(nvs) == 1: return nvs[0] raise Exception(f"More than one PypiPackage with {name}=={version}") - def fetch_wheel( - self, - environment=None, - fetched_filenames=None, - dest_dir=THIRDPARTY_DIR, - ): - """ - Download a binary wheel of this package matching the ``environment`` - Enviromnent constraints into ``dest_dir`` directory. - - Return the wheel filename if it was fetched, None otherwise. - - If the provided ``environment`` is None then the current Python - interpreter environment is used implicitly. Do not refetch wheel if - their name is in a provided ``fetched_filenames`` set. - """ - fetched_wheel_filename = None - if fetched_filenames is not None: - fetched_filenames = fetched_filenames - else: - fetched_filenames = set() - - supported_wheels = list(self.get_supported_wheels(environment)) - for wheel in supported_wheels: - - if wheel.filename not in fetched_filenames: - fetch_and_save_path_or_url( - filename=wheel.filename, - path_or_url=wheel.path_or_url, - dest_dir=dest_dir, - as_text=False, - ) - fetched_filenames.add(wheel.filename) - fetched_wheel_filename = wheel.filename - - # TODO: what if there is more than one? - break - - return fetched_wheel_filename - - def fetch_sdist(self, dest_dir=THIRDPARTY_DIR): - """ - Download the source distribution into `dest_dir` directory. Return the - fetched filename if it was fetched, False otherwise. - """ - if self.sdist: - assert self.sdist.filename - if TRACE: - print("Fetching source for package:", self.name, self.version) - fetch_and_save_path_or_url( - filename=self.sdist.filename, - dest_dir=dest_dir, - path_or_url=self.sdist.path_or_url, - as_text=False, - ) - if TRACE: - print(" --> file:", self.sdist.filename) - return self.sdist.filename - else: - print(f"Missing sdist for: {self.name}=={self.version}") - return False - - def delete_files(self, dest_dir=THIRDPARTY_DIR): - """ - Delete all PypiPackage files from `dest_dir` including wheels, sdist and - their ABOUT files. Note that we do not delete licenses since they can be - shared by several packages: therefore this would be done elsewhere in a - function that is aware of all used licenses. - """ - for to_delete in self.wheels + [self.sdist]: - if not to_delete: - continue - tdfn = to_delete.filename - for deletable in [tdfn, f"{tdfn}.ABOUT", f"{tdfn}.NOTICE"]: - target = os.path.join(dest_dir, deletable) - if os.path.exists(target): - print(f"Deleting outdated {target}") - fileutils.delete(target) - @classmethod - def get_dists(cls, paths_or_urls): + def dists_from_paths_or_urls(cls, paths_or_urls): """ Return a list of Distribution given a list of `paths_or_urls` to wheels or source distributions. @@ -1574,9 +1471,9 @@ def get_dists(cls, paths_or_urls): ... /home/foo/bitarray-0.8.1-cp36-cp36m-linux_x86_64.whl ... bitarray-0.8.1-cp36-cp36m-macosx_10_9_x86_64.macosx_10_10_x86_64.whl ... bitarray-0.8.1-cp36-cp36m-win_amd64.whl - ... httsp://example.com/bar/bitarray-0.8.1.tar.gz + ... https://example.com/bar/bitarray-0.8.1.tar.gz ... bitarray-0.8.1.tar.gz.ABOUT bit.LICENSE'''.split() - >>> result = list(PypiPackage.get_dists(paths_or_urls)) + >>> result = list(PypiPackage.dists_from_paths_or_urls(paths_or_urls)) >>> for r in results: ... r.filename = '' ... r.path_or_url = '' @@ -1590,18 +1487,28 @@ def get_dists(cls, paths_or_urls): ... Wheel(name='bitarray', version='0.8.1', build='', ... python_versions=['cp36'], abis=['cp36m'], ... platforms=['win_amd64']), + ... Sdist(name='bitarray', version='0.8.1'), ... Sdist(name='bitarray', version='0.8.1') ... ] >>> assert expected == result """ + dists = [] + if TRACE_DEEP: + print(" ###paths_or_urls:", paths_or_urls) installable = [f for f in paths_or_urls if f.endswith(EXTENSIONS_INSTALLABLE)] for path_or_url in installable: try: - yield Distribution.from_path_or_url(path_or_url) + dist = Distribution.from_path_or_url(path_or_url) + dists.append(dist) + if TRACE_DEEP: + print( + " ===> dists_from_paths_or_urls:", dist, "with URL:", dist.download_url + ) except InvalidDistributionFilename: - if TRACE: - print(f"Skipping invalid distribution from: {path_or_url}") + if TRACE_DEEP: + print(f" Skipping invalid distribution from: {path_or_url}") continue + return dists def get_distributions(self): """ @@ -1626,10 +1533,11 @@ class Environment: """ An Environment describes a target installation environment with its supported Python version, ABI, platform, implementation and related - attributes. We can use these to pass as `pip download` options and force - fetching only the subset of packages that match these Environment - constraints as opposed to the current running Python interpreter - constraints. + attributes. + + We can use these to pass as `pip download` options and force fetching only + the subset of packages that match these Environment constraints as opposed + to the current running Python interpreter constraints. """ python_version = attr.ib( @@ -1648,18 +1556,21 @@ class Environment: type=str, default="cp", metadata=dict(help="Python implementation supported by this environment."), + repr=False, ) abis = attr.ib( type=list, default=attr.Factory(list), - metadata=dict(help="List of ABI tags supported by this environment."), + metadata=dict(help="List of ABI tags supported by this environment."), + repr=False, ) platforms = attr.ib( type=list, default=attr.Factory(list), metadata=dict(help="List of platform tags supported by this environment."), + repr=False, ) @classmethod @@ -1677,18 +1588,20 @@ def from_pyver_and_os(cls, python_version, operating_system): def get_pip_cli_options(self): """ - Return a list of pip command line options for this environment. + Return a list of pip download command line options for this environment. """ options = [ "--python-version", self.python_version, "--implementation", self.implementation, - "--abi", - self.abi, ] + for abi in self.abis: + options.extend(["--abi", abi]) + for platform in self.platforms: options.extend(["--platform", platform]) + return options def tags(self): @@ -1704,7 +1617,6 @@ def tags(self): ) ) - ################################################################################ # # PyPI repo and link index for package wheels and sources @@ -1713,11 +1625,18 @@ def tags(self): @attr.attributes -class Repository: +class PypiSimpleRepository: """ - A PyPI or links Repository of Python packages: wheels, sdist, ABOUT, etc. + A PyPI repository of Python packages: wheels, sdist, etc. like the public + PyPI simple index. It is populated lazily based on requested packages names. """ + index_url = attr.ib( + type=str, + default=PYPI_SIMPLE_URL, + metadata=dict(help="Base PyPI simple URL for this index."), + ) + packages_by_normalized_name = attr.ib( type=dict, default=attr.Factory(lambda: defaultdict(list)), @@ -1730,126 +1649,157 @@ class Repository: metadata=dict(help="Mapping of {(name, version): package object} available in this repo"), ) - def get_links(self, *args, **kwargs): - raise NotImplementedError() - def get_versions(self, name): """ Return a list of all available PypiPackage version for this package name. The list may be empty. """ - raise NotImplementedError() + name = name and NameVer.normalize_name(name) + self._populate_links_and_packages(name) + return self.packages_by_normalized_name.get(name, []) + + def get_latest_version(self, name): + """ + Return the latest PypiPackage version for this package name or None. + """ + versions = self.get_versions(name) + return PypiPackage.get_latest_version(name, versions) def get_package(self, name, version): """ Return the PypiPackage with name and version or None. """ - raise NotImplementedError() + versions = self.get_versions(name) + if TRACE_DEEP: + print("PypiPackage.get_package:versions:", versions) + return PypiPackage.get_name_version(name, version, versions) - def get_latest_version(self, name): + def _fetch_links(self, name, _LINKS={}): """ - Return the latest PypiPackage version for this package name or None. + Return a list of download link URLs found in a PyPI simple index for package + name using the `index_url` of this repository. """ - raise NotImplementedError() + name = name and NameVer.normalize_name(name) + index_url = self.index_url + name = name and NameVer.normalize_name(name) + index_url = index_url.strip("/") + index_url = f"{index_url}/{name}" -@attr.attributes -class LinksRepository(Repository): - """ - Represents a simple links repository which is either a local directory with - Python wheels and sdist or a remote URL to an HTML with links to these. - (e.g. suitable for use with pip --find-links). - """ + if TRACE_DEEP: + print( + f" Finding links for {name!r} from PyPI index: {index_url} : cached?:", + index_url in _LINKS, + ) - path_or_url = attr.ib( - type=str, - default="", - metadata=dict(help="Package directory path or URL"), - ) + if index_url not in _LINKS: + text = fetch_content_from_path_or_url_through_cache(path_or_url=index_url, as_text=True) + links = collect_urls(text) + # TODO: keep sha256 + links = [l.partition("#sha256=") for l in links] + links = [url for url, _, _sha256 in links] + _LINKS[index_url] = [l for l in links if l.endswith(EXTENSIONS)] - links = attr.ib( - type=list, - default=attr.Factory(list), - metadata=dict(help="List of links available in this repo"), - ) + links = _LINKS[index_url] + if TRACE_DEEP: + print(f" Found links {links!r}") + return links - def __attrs_post_init__(self): - if not self.links: - self.links = get_paths_or_urls(links_url=self.path_or_url) - if not self.packages_by_normalized_name: - for p in PypiPackage.packages_from_many_paths_or_urls(paths_or_urls=self.links): - normalized_name = p.normalized_name - self.packages_by_normalized_name[normalized_name].append(p) - self.packages_by_normalized_name_version[(normalized_name, p.version)] = p + def _populate_links_and_packages(self, name): + name = name and NameVer.normalize_name(name) - def get_links(self, *args, **kwargs): - return self.links or [] + if TRACE_DEEP: + print("PypiPackage._populate_links_and_packages:name:", name) - def get_versions(self, name): - name = name and NameVer.normalize_name(name) - return self.packages_by_normalized_name.get(name, []) + links = self._fetch_links(name) + packages = list(PypiPackage.packages_from_many_paths_or_urls(paths_or_urls=links)) - def get_latest_version(self, name): - return PypiPackage.get_latest_version(name, self.get_versions(name)) + if TRACE_DEEP: + print("PypiPackage._populate_links_and_packages:packages:", packages) - def get_package(self, name, version): - return PypiPackage.get_name_version(name, version, self.get_versions(name)) + self.packages_by_normalized_name[name] = packages + + for p in packages: + name = name and NameVer.normalize_name(p.name) + self.packages_by_normalized_name_version[(name, p.version)] = p @attr.attributes -class PypiRepository(Repository): +class LinksRepository: """ - Represents the public PyPI simple index. - It is populated lazily based on requested packages names + Represents a simple links repository such an HTTP directory listing or a + page with links. """ - simple_url = attr.ib( + url = attr.ib( type=str, - default=PYPI_SIMPLE_URL, - metadata=dict(help="Base PyPI simple URL for this index."), + default="", + metadata=dict(help="Links directory URL"), ) - links_by_normalized_name = attr.ib( - type=dict, - default=attr.Factory(lambda: defaultdict(list)), - metadata=dict(help="Mapping of {package name: [links]} available in this repo"), + links = attr.ib( + type=list, + default=attr.Factory(list), + metadata=dict(help="List of links available in this repo"), ) - def _fetch_links(self, name): - name = name and NameVer.normalize_name(name) - return find_pypi_links(name=name, simple_url=self.simple_url) + def __attrs_post_init__(self): + if not self.links: + self.links = self.find_links() - def _populate_links_and_packages(self, name): - name = name and NameVer.normalize_name(name) - if name in self.links_by_normalized_name: - return + def find_links(self): + """ + Return a list of link URLs found in the HTML page at `self.url` + """ + links_url = self.url + if TRACE_DEEP: + print(f"Finding links from: {links_url}") + plinks_url = urllib.parse.urlparse(links_url) + base_url = urllib.parse.SplitResult( + plinks_url.scheme, plinks_url.netloc, "", "", "" + ).geturl() - links = self._fetch_links(name) - self.links_by_normalized_name[name] = links + if TRACE_DEEP: + print(f"Base URL {base_url}") - packages = list(PypiPackage.packages_from_many_paths_or_urls(paths_or_urls=links)) - self.packages_by_normalized_name[name] = packages + text = fetch_content_from_path_or_url_through_cache( + path_or_url=links_url, + as_text=True, + ) - for p in packages: - name = name and NameVer.normalize_name(p.name) - self.packages_by_normalized_name_version[(name, p.version)] = p + links = [] + for link in collect_urls(text): + if not link.endswith(EXTENSIONS): + continue - def get_links(self, name, *args, **kwargs): - name = name and NameVer.normalize_name(name) - self._populate_links_and_packages(name) - return self.links_by_normalized_name.get(name, []) + plink = urllib.parse.urlsplit(link) - def get_versions(self, name): - name = name and NameVer.normalize_name(name) - self._populate_links_and_packages(name) - return self.packages_by_normalized_name.get(name, []) + if plink.scheme: + # full URL kept as-is + url = link - def get_latest_version(self, name): - return PypiPackage.get_latest_version(name, self.get_versions(name)) + if plink.path.startswith("/"): + # absolute link + url = f"{base_url}{link}" - def get_package(self, name, version): - return PypiPackage.get_name_version(name, version, self.get_versions(name)) + else: + # relative link + url = f"{links_url}/{link}" + + if TRACE_DEEP: + print(f"Adding URL: {url}") + links.append(url) + + if TRACE: + print(f"Found {len(links)} links at {links_url}") + return links + + @classmethod + def from_url(cls, url=ABOUT_BASE_URL, _LINKS_REPO={}): + if url not in _LINKS_REPO: + _LINKS_REPO[url] = cls(url=url) + return _LINKS_REPO[url] ################################################################################ # Globals for remote repos to be lazily created and cached on first use for the @@ -1862,52 +1812,27 @@ def get_local_packages(directory=THIRDPARTY_DIR): Return the list of all PypiPackage objects built from a local directory. Return an empty list if the package cannot be found. """ - return list(PypiPackage.packages_from_one_path_or_url(path_or_url=directory)) - - -def get_local_repo(directory=THIRDPARTY_DIR): - return LinksRepository(path_or_url=directory) - + return list(PypiPackage.packages_from_dir(directory=directory)) -_REMOTE_REPO = None +def get_pypi_repo(index_url, _PYPI_REPO={}): + if index_url not in _PYPI_REPO: + _PYPI_REPO[index_url] = PypiSimpleRepository(index_url=index_url) + return _PYPI_REPO[index_url] -def get_remote_repo(remote_links_url=REMOTE_LINKS_URL): - global _REMOTE_REPO - if not _REMOTE_REPO: - _REMOTE_REPO = LinksRepository(path_or_url=remote_links_url) - return _REMOTE_REPO - -def get_remote_package(name, version, remote_links_url=REMOTE_LINKS_URL): +def get_pypi_package(name, version, index_url, verbose=TRACE_DEEP): """ Return a PypiPackage or None. """ try: - return get_remote_repo(remote_links_url).get_package(name, version) - except RemoteNotFetchedException as e: - print(f"Failed to fetch remote package info: {e}") - - -_PYPI_REPO = None - - -def get_pypi_repo(pypi_simple_url=PYPI_SIMPLE_URL): - global _PYPI_REPO - if not _PYPI_REPO: - _PYPI_REPO = PypiRepository(simple_url=pypi_simple_url) - return _PYPI_REPO - + package = get_pypi_repo(index_url).get_package(name, version) + if verbose: + print(f" get_pypi_package: {name} @ {version} info from {index_url}: {package}") + return package -def get_pypi_package(name, version, pypi_simple_url=PYPI_SIMPLE_URL): - """ - Return a PypiPackage or None. - """ - try: - return get_pypi_repo(pypi_simple_url).get_package(name, version) except RemoteNotFetchedException as e: - print(f"Failed to fetch remote package info: {e}") - + print(f"Failed to fetch PyPI package {name} @ {version} info from {index_url}: {e}") ################################################################################ # @@ -1936,8 +1861,8 @@ def get(self, path_or_url, as_text=True): Get a file from a `path_or_url` through the cache. `path_or_url` can be a path or a URL to a file. """ - filename = os.path.basename(path_or_url.strip("/")) - cached = os.path.join(self.directory, filename) + cache_key = quote_plus(path_or_url.strip("/")) + cached = os.path.join(self.directory, cache_key) if not os.path.exists(cached): content = get_file_content(path_or_url=path_or_url, as_text=as_text) @@ -1948,32 +1873,23 @@ def get(self, path_or_url, as_text=True): else: return get_local_file_content(path=cached, as_text=as_text) - def put(self, filename, content): - """ - Put in the cache the `content` of `filename`. - """ - cached = os.path.join(self.directory, filename) - wmode = "wb" if isinstance(content, bytes) else "w" - with open(cached, wmode) as fo: - fo.write(content) - def get_file_content(path_or_url, as_text=True): """ Fetch and return the content at `path_or_url` from either a local path or a remote URL. Return the content as bytes is `as_text` is False. """ - if path_or_url.startswith("file://") or ( - path_or_url.startswith("/") and os.path.exists(path_or_url) - ): - return get_local_file_content(path=path_or_url, as_text=as_text) - - elif path_or_url.startswith("https://"): + if path_or_url.startswith("https://"): if TRACE: print(f"Fetching: {path_or_url}") _headers, content = get_remote_file_content(url=path_or_url, as_text=as_text) return content + elif path_or_url.startswith("file://") or ( + path_or_url.startswith("/") and os.path.exists(path_or_url) + ): + return get_local_file_content(path=path_or_url, as_text=as_text) + else: raise Exception(f"Unsupported URL scheme: {path_or_url}") @@ -2016,6 +1932,7 @@ def get_remote_file_content( # using a GET with stream=True ensure we get the the final header from # several redirects and that we can ignore content there. A HEAD request may # not get us this last header + print(f" DOWNLOADING {url}") with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: status = response.status_code if status != requests.codes.ok: # NOQA @@ -2039,76 +1956,11 @@ def get_remote_file_content( return response.headers, response.text if as_text else response.content -def get_url_content_if_modified( - url, - md5, - _delay=0, -): - """ - Return fetched content bytes at `url` or None if the md5 has not changed. - Retries multiple times to fetch if there is a HTTP 429 throttling response - and this with an increasing delay. - """ - time.sleep(_delay) - headers = None - if md5: - etag = f'"{md5}"' - headers = {"If-None-Match": f"{etag}"} - - # using a GET with stream=True ensure we get the the final header from - # several redirects and that we can ignore content there. A HEAD request may - # not get us this last header - with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: - status = response.status_code - if status == requests.codes.too_many_requests and _delay < 20: # NOQA - # too many requests: start waiting with some exponential delay - _delay = (_delay * 2) or 1 - return get_url_content_if_modified(url=url, md5=md5, _delay=_delay) - - elif status == requests.codes.not_modified: # NOQA - # all is well, the md5 is the same - return None - - elif status != requests.codes.ok: # NOQA - raise RemoteNotFetchedException(f"Failed HTTP request from {url} with {status}") - - return response.content - - -def get_remote_headers(url): - """ - Fetch and return a mapping of HTTP headers of `url`. - """ - headers, _content = get_remote_file_content(url, headers_only=True) - return headers - - -def fetch_and_save_filename_from_paths_or_urls( - filename, - paths_or_urls, - dest_dir=THIRDPARTY_DIR, +def fetch_content_from_path_or_url_through_cache( + path_or_url, as_text=True, + cache=Cache(), ): - """ - Return the content from fetching the `filename` file name found in the - `paths_or_urls` list of URLs or paths and save to `dest_dir`. Raise an - Exception on errors. Treats the content as text if `as_text` is True - otherwise as binary. - """ - path_or_url = get_link_for_filename( - filename=filename, - paths_or_urls=paths_or_urls, - ) - - return fetch_and_save_path_or_url( - filename=filename, - dest_dir=dest_dir, - path_or_url=path_or_url, - as_text=as_text, - ) - - -def fetch_content_from_path_or_url_through_cache(path_or_url, as_text=True, cache=Cache()): """ Return the content from fetching at path or URL. Raise an Exception on errors. Treats the content as text if as_text is True otherwise as treat as @@ -2118,423 +1970,90 @@ def fetch_content_from_path_or_url_through_cache(path_or_url, as_text=True, cach Note: the `cache` argument is a global, though it does not really matter since it does not hold any state which is only kept on disk. """ - if cache: - return cache.get(path_or_url=path_or_url, as_text=as_text) - else: - return get_file_content(path_or_url=path_or_url, as_text=as_text) + return cache.get(path_or_url=path_or_url, as_text=as_text) -def fetch_and_save_path_or_url(filename, dest_dir, path_or_url, as_text=True, through_cache=True): +def fetch_and_save_path_or_url( + filename, + dest_dir, + path_or_url, + as_text=True, +): """ Return the content from fetching the `filename` file name at URL or path and save to `dest_dir`. Raise an Exception on errors. Treats the content as text if as_text is True otherwise as treat as binary. """ - if through_cache: - content = fetch_content_from_path_or_url_through_cache(path_or_url, as_text) - else: - content = fetch_content_from_path_or_url_through_cache(path_or_url, as_text, cache=None) - + content = fetch_content_from_path_or_url_through_cache( + path_or_url=path_or_url, + as_text=as_text, + ) output = os.path.join(dest_dir, filename) wmode = "w" if as_text else "wb" with open(output, wmode) as fo: fo.write(content) return content - ################################################################################ -# -# Sync and fix local thirdparty directory for various issues and gaps -# +# Requirements processing ################################################################################ -def fetch_missing_sources(dest_dir=THIRDPARTY_DIR): - """ - Given a thirdparty dir, fetch missing source distributions from our remote - repo or PyPI. Return a list of (name, version) tuples for source - distribution that were not found - """ - not_found = [] - local_packages = get_local_packages(directory=dest_dir) - remote_repo = get_remote_repo() - pypi_repo = get_pypi_repo() - - for package in local_packages: - if not package.sdist: - print(f"Finding sources for: {package.name}=={package.version}: ", end="") - try: - pypi_package = pypi_repo.get_package(name=package.name, version=package.version) - - if pypi_package and pypi_package.sdist: - print(f"Fetching sources from Pypi") - pypi_package.fetch_sdist(dest_dir=dest_dir) - continue - else: - remote_package = remote_repo.get_package( - name=package.name, version=package.version - ) - - if remote_package and remote_package.sdist: - print(f"Fetching sources from Remote") - remote_package.fetch_sdist(dest_dir=dest_dir) - continue - - except RemoteNotFetchedException as e: - print(f"Failed to fetch remote package info: {e}") - - print(f"No sources found") - not_found.append( - ( - package.name, - package.version, - ) - ) - - return not_found - - -def fetch_missing_wheels( - python_versions=PYTHON_VERSIONS, - operating_systems=PLATFORMS_BY_OS, - dest_dir=THIRDPARTY_DIR, +def get_required_remote_packages( + requirements_file="requirements.txt", + index_url=PYPI_SIMPLE_URL, ): """ - Given a thirdparty dir fetch missing wheels for all known combos of Python - versions and OS. Return a list of tuple (Package, Environment) for wheels - that were not found locally or remotely. + Yield tuple of (name, version, PypiPackage) for packages listed in the + `requirements_file` requirements file and found in the PyPI index + ``index_url`` URL. """ - local_packages = get_local_packages(directory=dest_dir) - evts = itertools.product(python_versions, operating_systems) - environments = [Environment.from_pyver_and_os(pyv, os) for pyv, os in evts] - packages_and_envts = itertools.product(local_packages, environments) - - not_fetched = [] - fetched_filenames = set() - for package, envt in packages_and_envts: + required_name_versions = load_requirements(requirements_file=requirements_file) + return get_required_packages(required_name_versions=required_name_versions, index_url=index_url) - filename = package.fetch_wheel( - environment=envt, - fetched_filenames=fetched_filenames, - dest_dir=dest_dir, - ) - if filename: - fetched_filenames.add(filename) - else: - not_fetched.append( - ( - package, - envt, - ) - ) - - return not_fetched - - -def build_missing_wheels( - packages_and_envts, - build_remotely=False, - with_deps=False, - dest_dir=THIRDPARTY_DIR, - remote_build_log_file=None, +def get_required_packages( + required_name_versions, + index_url=PYPI_SIMPLE_URL, ): """ - Build all wheels in a list of tuple (Package, Environment) and save in - `dest_dir`. Return a list of tuple (Package, Environment), and a list of - built wheel filenames. - """ - - not_built = [] - built_filenames = [] - - packages_and_envts = itertools.groupby(sorted(packages_and_envts), key=operator.itemgetter(0)) - - for package, pkg_envts in packages_and_envts: - - envts = [envt for _pkg, envt in pkg_envts] - python_versions = sorted(set(e.python_version for e in envts)) - operating_systems = sorted(set(e.operating_system for e in envts)) - built = None - try: - built = build_wheels( - requirements_specifier=package.specifier, - with_deps=with_deps, - build_remotely=build_remotely, - python_versions=python_versions, - operating_systems=operating_systems, - verbose=TRACE, - dest_dir=dest_dir, - remote_build_log_file=remote_build_log_file, - ) - print(".") - except Exception as e: - import traceback - - print("#############################################################") - print("############# WHEEL BUILD FAILED ######################") - traceback.print_exc() - print() - print("#############################################################") - - if not built: - for envt in pkg_envts: - not_built.append((package, envt)) - else: - for bfn in built: - print(f" --> Built wheel: {bfn}") - built_filenames.append(bfn) - - return not_built, built_filenames - - -################################################################################ -# -# Functions to handle remote or local repo used to "find-links" -# -################################################################################ - - -def get_paths_or_urls(links_url): - if links_url.startswith("https:"): - paths_or_urls = find_links_from_release_url(links_url) - else: - paths_or_urls = find_links_from_dir(links_url) - return paths_or_urls - - -def find_links_from_dir(directory=THIRDPARTY_DIR): - """ - Return a list of path to files in `directory` for any file that ends with - any of the extension in the list of `extensions` strings. - """ - base = os.path.abspath(directory) - files = [os.path.join(base, f) for f in os.listdir(base) if f.endswith(EXTENSIONS)] - return files - - -get_links = re.compile('href="([^"]+)"').findall - - -def find_links_from_release_url(links_url=REMOTE_LINKS_URL): - """ - Return a list of download link URLs found in the HTML page at `links_url` - URL that starts with the `prefix` string and ends with any of the extension - in the list of `extensions` strings. Use the `base_url` to prefix the links. + Yield tuple of (name, version) or a PypiPackage for package name/version + listed in the ``required_name_versions`` list and found in the PyPI index + ``index_url`` URL. """ if TRACE: - print(f"Finding links for {links_url}") - - plinks_url = urllib.parse.urlparse(links_url) - - base_url = urllib.parse.SplitResult(plinks_url.scheme, plinks_url.netloc, "", "", "").geturl() - - if TRACE: - print(f"Base URL {base_url}") - - _headers, text = get_remote_file_content(links_url) - links = [] - for link in get_links(text): - if not link.endswith(EXTENSIONS): - continue + print("get_required_packages", index_url) - plink = urllib.parse.urlsplit(link) - - if plink.scheme: - # full URL kept as-is - url = link - - if plink.path.startswith("/"): - # absolute link - url = f"{base_url}{link}" - - else: - # relative link - url = f"{links_url}/{link}" + repo = get_pypi_repo(index_url=index_url) + for name, version in required_name_versions: if TRACE: - print(f"Adding URL: {url}") - - links.append(url) - - if TRACE: - print(f"Found {len(links)} links at {links_url}") - return links - - -def find_pypi_links(name, simple_url=PYPI_SIMPLE_URL): - """ - Return a list of download link URLs found in a PyPI simple index for package name. - with the list of `extensions` strings. Use the `simple_url` PyPI url. - """ - if TRACE: - print(f"Finding links for {simple_url}") - - name = name and NameVer.normalize_name(name) - simple_url = simple_url.strip("/") - simple_url = f"{simple_url}/{name}" - - _headers, text = get_remote_file_content(simple_url) - links = get_links(text) - # TODO: keep sha256 - links = [l.partition("#sha256=") for l in links] - links = [url for url, _, _sha256 in links] - links = [l for l in links if l.endswith(EXTENSIONS)] - return links - - -def get_link_for_filename(filename, paths_or_urls): - """ - Return a link for `filename` found in the `links` list of URLs or paths. Raise an - exception if no link is found or if there are more than one link for that - file name. - """ - path_or_url = [l for l in paths_or_urls if l.endswith(f"/{filename}")] - if not path_or_url: - raise Exception(f"Missing link to file: {filename}") - if not len(path_or_url) == 1: - raise Exception(f"Multiple links to file: {filename}: \n" + "\n".join(path_or_url)) - return path_or_url[0] - + print(" get_required_packages: name:", name, "version:", version) + yield repo.get_package(name, version) ################################################################################ -# -# Requirements processing -# +# Functions to update or fetch ABOUT and license files ################################################################################ -class MissingRequirementException(Exception): - pass - - -def get_required_packages(required_name_versions): - """ - Return a tuple of (remote packages, PyPI packages) where each is a mapping - of {(name, version): PypiPackage} for packages listed in the - `required_name_versions` list of (name, version) tuples. Raise a - MissingRequirementException with a list of missing (name, version) if a - requirement cannot be satisfied remotely or in PyPI. - """ - remote_repo = get_remote_repo() - - remote_packages = { - (name, version): remote_repo.get_package(name, version) - for name, version in required_name_versions - } - - pypi_repo = get_pypi_repo() - pypi_packages = { - (name, version): pypi_repo.get_package(name, version) - for name, version in required_name_versions - } - - # remove any empty package (e.g. that do not exist in some place) - remote_packages = {nv: p for nv, p in remote_packages.items() if p} - pypi_packages = {nv: p for nv, p in pypi_packages.items() if p} - - # check that we are not missing any - repos_name_versions = set(remote_packages.keys()) | set(pypi_packages.keys()) - missing_name_versions = required_name_versions.difference(repos_name_versions) - if missing_name_versions: - raise MissingRequirementException(sorted(missing_name_versions)) - - return remote_packages, pypi_packages - - -def get_required_remote_packages( - requirements_file="requirements.txt", - force_pinned=True, - remote_links_url=REMOTE_LINKS_URL, +def clean_about_files( + dest_dir=THIRDPARTY_DIR, ): """ - Yield tuple of (name, version, PypiPackage) for packages listed in the - `requirements_file` requirements file and found in the PyPI-like link repo - ``remote_links_url`` if this is a URL. Treat this ``remote_links_url`` as a - local directory path to a wheels directory if this is not a a URL. - """ - required_name_versions = load_requirements( - requirements_file=requirements_file, - force_pinned=force_pinned, - ) - - if remote_links_url.startswith("https://"): - repo = get_remote_repo(remote_links_url=remote_links_url) - else: - # a local path - assert os.path.exists(remote_links_url), f"Path does not exist: {remote_links_url}" - repo = get_local_repo(directory=remote_links_url) - - for name, version in required_name_versions: - if version: - yield name, version, repo.get_package(name, version) - else: - yield name, version, repo.get_latest_version(name) - - -def update_requirements(name, version=None, requirements_file="requirements.txt"): - """ - Upgrade or add `package_name` with `new_version` to the `requirements_file` - requirements file. Write back requirements sorted with name and version - canonicalized. Note: this cannot deal with hashed or unpinned requirements. - Do nothing if the version already exists as pinned. + Given a thirdparty dir, clean ABOUT files """ - normalized_name = NameVer.normalize_name(name) - - is_updated = False - updated_name_versions = [] - for existing_name, existing_version in load_requirements(requirements_file, force_pinned=False): - - existing_normalized_name = NameVer.normalize_name(existing_name) - - if normalized_name == existing_normalized_name: - if version != existing_version: - is_updated = True - updated_name_versions.append( - ( - existing_normalized_name, - existing_version, - ) - ) - - if is_updated: - updated_name_versions = sorted(updated_name_versions) - nvs = "\n".join(f"{name}=={version}" for name, version in updated_name_versions) - - with open(requirements_file, "w") as fo: - fo.write(nvs) - - -def hash_requirements(dest_dir=THIRDPARTY_DIR, requirements_file="requirements.txt"): - """ - Hash all the requirements found in the `requirements_file` - requirements file based on distributions available in `dest_dir` - """ - local_repo = get_local_repo(directory=dest_dir) - packages_by_normalized_name_version = local_repo.packages_by_normalized_name_version - hashed = [] - for name, version in load_requirements(requirements_file, force_pinned=True): - package = packages_by_normalized_name_version.get((name, version)) - if not package: - raise Exception(f"Missing required package {name}=={version}") - hashed.append(package.specifier_with_hashes) - - with open(requirements_file, "w") as fo: - fo.write("\n".join(hashed)) - + local_packages = get_local_packages(directory=dest_dir) + for local_package in local_packages: + for local_dist in local_package.get_distributions(): + local_dist.load_about_data(dest_dir=dest_dir) + local_dist.set_checksums(dest_dir=dest_dir) -################################################################################ -# -# Functions to update or fetch ABOUT and license files -# -################################################################################ + if "classifiers" in local_dist.extra_data: + local_dist.extra_data.pop("classifiers", None) + local_dist.save_about_and_notice_files(dest_dir) -def add_fetch_or_update_about_and_license_files( - dest_dir=THIRDPARTY_DIR, - include_remote=True, - strip_classifiers=False, -): +def fetch_abouts_and_licenses(dest_dir=THIRDPARTY_DIR): """ Given a thirdparty dir, add missing ABOUT. LICENSE and NOTICE files using best efforts: @@ -2544,32 +2063,28 @@ def add_fetch_or_update_about_and_license_files( - derive from existing distribution with same name and latest version that would have such ABOUT file - extract ABOUT file data from distributions PKGINFO or METADATA files - - TODO: make API calls to fetch package data from DejaCode - - The process consists in load and iterate on every package distributions, - collect data and then acsk to save. """ - local_packages = get_local_packages(directory=dest_dir) - local_repo = get_local_repo(directory=dest_dir) - - remote_repo = get_remote_repo() - def get_other_dists(_package, _dist): """ - Return a list of all the dists from package that are not the `dist` object + Return a list of all the dists from `_package` that are not the `_dist` + object """ return [d for d in _package.get_distributions() if d != _dist] + selfhosted_repo = get_pypi_repo(index_url=ABOUT_PYPI_SIMPLE_URL) + local_packages = get_local_packages(directory=dest_dir) + packages_by_name = defaultdict(list) + for local_package in local_packages: + distributions = list(local_package.get_distributions()) + distribution = distributions[0] + packages_by_name[distribution.name].append(local_package) + for local_package in local_packages: for local_dist in local_package.get_distributions(): local_dist.load_about_data(dest_dir=dest_dir) local_dist.set_checksums(dest_dir=dest_dir) - if strip_classifiers and "classifiers" in local_dist.extra_data: - local_dist.extra_data.pop("classifiers", None) - local_dist.save_about_and_notice_files(dest_dir) - # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) @@ -2588,16 +2103,16 @@ def get_other_dists(_package, _dist): local_dist.fetch_license_files(dest_dir=dest_dir) continue - # try to get a latest version of the same package that is not our version + # try to get another version of the same package that is not our version other_local_packages = [ p - for p in local_repo.get_versions(local_package.name) + for p in packages_by_name[local_package.name] if p.version != local_package.version ] - latest_local_version = other_local_packages and other_local_packages[-1] - if latest_local_version: - latest_local_dists = list(latest_local_version.get_distributions()) + other_local_version = other_local_packages and other_local_packages[-1] + if other_local_version: + latest_local_dists = list(other_local_version.get_distributions()) for latest_local_dist in latest_local_dists: latest_local_dist.load_about_data(dest_dir=dest_dir) if not latest_local_dist.has_key_metadata(): @@ -2615,9 +2130,35 @@ def get_other_dists(_package, _dist): local_dist.fetch_license_files(dest_dir=dest_dir) continue - if include_remote: - # lets try to fetch remotely - local_dist.load_remote_about_data() + # lets try to fetch remotely + local_dist.load_remote_about_data() + + # if has key data we may look to improve later, but we can move on + if local_dist.has_key_metadata(): + local_dist.save_about_and_notice_files(dest_dir=dest_dir) + local_dist.fetch_license_files(dest_dir=dest_dir) + continue + + # try to get a latest version of the same package that is not our version + other_remote_packages = [ + p + for p in selfhosted_repo.get_versions(local_package.name) + if p.version != local_package.version + ] + + latest_version = other_remote_packages and other_remote_packages[-1] + if latest_version: + latest_dists = list(latest_version.get_distributions()) + for remote_dist in latest_dists: + remote_dist.load_remote_about_data() + if not remote_dist.has_key_metadata(): + # there is not much value to get other data if we are missing the key ones + continue + else: + local_dist.update_from_other_dist(remote_dist) + # if has key data we may look to improve later, but we can move on + if local_dist.has_key_metadata(): + break # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): @@ -2625,33 +2166,6 @@ def get_other_dists(_package, _dist): local_dist.fetch_license_files(dest_dir=dest_dir) continue - # try to get a latest version of the same package that is not our version - other_remote_packages = [ - p - for p in remote_repo.get_versions(local_package.name) - if p.version != local_package.version - ] - - latest_version = other_remote_packages and other_remote_packages[-1] - if latest_version: - latest_dists = list(latest_version.get_distributions()) - for remote_dist in latest_dists: - remote_dist.load_remote_about_data() - if not remote_dist.has_key_metadata(): - # there is not much value to get other data if we are missing the key ones - continue - else: - local_dist.update_from_other_dist(remote_dist) - # if has key data we may look to improve later, but we can move on - if local_dist.has_key_metadata(): - break - - # if has key data we may look to improve later, but we can move on - if local_dist.has_key_metadata(): - local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files(dest_dir=dest_dir) - continue - # try to get data from pkginfo (no license though) local_dist.load_pkginfo_data(dest_dir=dest_dir) @@ -2661,15 +2175,12 @@ def get_other_dists(_package, _dist): lic_errs = local_dist.fetch_license_files(dest_dir) - # TODO: try to get data from dejacode - if not local_dist.has_key_metadata(): print(f"Unable to add essential ABOUT data for: {local_dist}") if lic_errs: lic_errs = "\n".join(lic_errs) print(f"Failed to fetch some licenses:: {lic_errs}") - ################################################################################ # # Functions to build new Python wheels including native on multiple OSes @@ -2680,9 +2191,9 @@ def get_other_dists(_package, _dist): def call(args, verbose=TRACE): """ Call args in a subprocess and display output on the fly if ``trace`` is True. - Return or raise stdout, stderr, returncode + Return a tuple of (returncode, stdout, stderr) """ - if TRACE: + if TRACE_DEEP: print("Calling:", " ".join(args)) with subprocess.Popen( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8" @@ -2700,312 +2211,78 @@ def call(args, verbose=TRACE): stdout, stderr = process.communicate() if not stdout.strip(): stdout = "\n".join(stdouts) - - returncode = process.returncode - - if returncode == 0: - return returncode, stdout, stderr - else: - raise Exception(returncode, stdout, stderr) - - -def add_or_upgrade_built_wheels( - name, - version=None, - python_versions=PYTHON_VERSIONS, - operating_systems=PLATFORMS_BY_OS, - dest_dir=THIRDPARTY_DIR, - build_remotely=False, - with_deps=False, - verbose=TRACE, - remote_build_log_file=None, -): - """ - Add or update package `name` and `version` as a binary wheel saved in - `dest_dir`. Use the latest version if `version` is None. Return the a list - of the collected, fetched or built wheel file names or an empty list. - - Use the provided lists of `python_versions` (e.g. "36", "39") and - `operating_systems` (e.g. linux, windows or macos) to decide which specific - wheel to fetch or build. - - Include wheels for all dependencies if `with_deps` is True. - Build remotely is `build_remotely` is True. - Do not wait for build completion and log to ``remote_build_log_file`` - file path if provided. - """ - assert name, "Name is required" - ver = version and f"=={version}" or "" - print(f"\nAdding wheels for package: {name}{ver}") - - if verbose: - print("python_versions:", python_versions) - print("operating_systems:", operating_systems) - - wheel_filenames = [] - # a mapping of {req specifier: {mapping build_wheels kwargs}} - wheels_to_build = {} - for python_version, operating_system in itertools.product(python_versions, operating_systems): - print( - f" Adding wheels for package: {name}{ver} on {python_version,} and {operating_system}" - ) - environment = Environment.from_pyver_and_os(python_version, operating_system) - - # Check if requested wheel already exists locally for this version - local_repo = get_local_repo(directory=dest_dir) - local_package = local_repo.get_package(name=name, version=version) - - has_local_wheel = False - if version and local_package: - for wheel in local_package.get_supported_wheels(environment): - has_local_wheel = True - wheel_filenames.append(wheel.filename) - break - if has_local_wheel: - print(f" local wheel exists: {wheel.filename}") - continue - - if not version: - pypi_package = get_pypi_repo().get_latest_version(name) - version = pypi_package.version - - # Check if requested wheel already exists remotely or in Pypi for this version - wheel_filename = fetch_package_wheel( - name=name, version=version, environment=environment, dest_dir=dest_dir - ) - if verbose: - print(" fetching package wheel:", wheel_filename) - if wheel_filename: - wheel_filenames.append(wheel_filename) - - # the wheel is not available locally, remotely or in Pypi - # we need to build binary from sources - requirements_specifier = f"{name}=={version}" - to_build = wheels_to_build.get(requirements_specifier) - if to_build: - to_build["python_versions"].append(python_version) - to_build["operating_systems"].append(operating_system) - else: - wheels_to_build[requirements_specifier] = dict( - requirements_specifier=requirements_specifier, - python_versions=[python_version], - operating_systems=[operating_system], - dest_dir=dest_dir, - build_remotely=build_remotely, - with_deps=with_deps, - verbose=verbose, - remote_build_log_file=remote_build_log_file, - ) - - for build_wheels_kwargs in wheels_to_build.values(): - bwheel_filenames = build_wheels(**build_wheels_kwargs) - wheel_filenames.extend(bwheel_filenames) - - return sorted(set(wheel_filenames)) - - -def build_wheels( - requirements_specifier, - python_versions=PYTHON_VERSIONS, - operating_systems=PLATFORMS_BY_OS, - dest_dir=THIRDPARTY_DIR, - build_remotely=False, - with_deps=False, - verbose=False, - remote_build_log_file=None, -): - """ - Given a pip `requirements_specifier` string (such as package names or as - name==version), build the corresponding binary wheel(s) for all - `python_versions` and `operating_systems` combinations and save them - back in `dest_dir` and return a list of built wheel file names. - - Include wheels for all dependencies if `with_deps` is True. - - First try to build locally to process pure Python wheels, and fall back to - build remotey on all requested Pythons and operating systems. - - Do not wait for build completion and log to ``remote_build_log_file`` - file path if provided. - """ - all_pure, builds = build_wheels_locally_if_pure_python( - requirements_specifier=requirements_specifier, - with_deps=with_deps, - verbose=verbose, - dest_dir=dest_dir, - ) - for local_build in builds: - print(f"Built wheel: {local_build}") - - if all_pure: - return builds - - if build_remotely: - remote_builds = build_wheels_remotely_on_multiple_platforms( - requirements_specifier=requirements_specifier, - with_deps=with_deps, - python_versions=python_versions, - operating_systems=operating_systems, - verbose=verbose, - dest_dir=dest_dir, - remote_build_log_file=remote_build_log_file, - ) - builds.extend(remote_builds) - - return builds - - -def build_wheels_remotely_on_multiple_platforms( - requirements_specifier, - with_deps=False, - python_versions=PYTHON_VERSIONS, - operating_systems=PLATFORMS_BY_OS, - verbose=False, - dest_dir=THIRDPARTY_DIR, - remote_build_log_file=None, -): - """ - Given pip `requirements_specifier` string (such as package names or as - name==version), build the corresponding binary wheel(s) including wheels for - all dependencies for all `python_versions` and `operating_systems` - combinations and save them back in `dest_dir` and return a list of built - wheel file names. - - Do not wait for build completion and log to ``remote_build_log_file`` - file path if provided. - """ - check_romp_is_configured() - pyos_options = get_romp_pyos_options(python_versions, operating_systems) - deps = "" if with_deps else "--no-deps" - verbose = "--verbose" if verbose else "" - - if remote_build_log_file: - # zero seconds, no wait, log to file instead - wait_build_for = "0" - else: - wait_build_for = DEFAULT_ROMP_BUILD_WAIT - - romp_args = [ - "romp", - "--interpreter", - "cpython", - "--architecture", - "x86_64", - "--check-period", - wait_build_for, # in seconds - ] - - if remote_build_log_file: - romp_args += ["--build-log-file", remote_build_log_file] - - romp_args += pyos_options + [ - "--artifact-paths", - "*.whl", - "--artifact", - "artifacts.tar.gz", - "--command", - f"python -m pip {verbose} install --user --upgrade pip setuptools wheel; " - f"python -m pip {verbose} wheel {deps} {requirements_specifier}", - ] - - if verbose: - romp_args.append("--verbose") - - print(f"Building wheels for: {requirements_specifier}") - print(f"Using command:", " ".join(romp_args)) - call(romp_args) - wheel_filenames = [] - if not remote_build_log_file: - wheel_filenames = extract_tar("artifacts.tar.gz", dest_dir) - for wfn in wheel_filenames: - print(f" built wheel: {wfn}") - return wheel_filenames + return process.returncode, stdout, stderr -def fetch_remotely_built_wheels( - remote_build_log_file, +def download_wheels_with_pip( + requirements_specifiers=tuple(), + requirements_files=tuple(), + environment=None, dest_dir=THIRDPARTY_DIR, - no_wait=False, - verbose=False, + index_url=PYPI_SIMPLE_URL, + links_url=ABOUT_LINKS_URL, ): """ - Given a ``remote_build_log_file`` file path with a JSON lines log of a - remote build, fetch the built wheels and move them to ``dest_dir``. Return a - list of built wheel file names. - """ - wait = "0" if no_wait else DEFAULT_ROMP_BUILD_WAIT # in seconds - - romp_args = [ - "romp-fetch", - "--build-log-file", - remote_build_log_file, - "--check-period", - wait, + Fetch binary wheel(s) using pip for the ``envt`` Environment given a list of + pip ``requirements_files`` and a list of ``requirements_specifiers`` string + (such as package names or as name==version). + Return a tuple of (list of downloaded files, error string). + Do NOT fail on errors, but return an error message on failure. + """ + + cli_args = [ + "pip", + "download", + "--only-binary", + ":all:", + "--dest", + dest_dir, + "--index-url", + index_url, + "--find-links", + links_url, + "--no-color", + "--progress-bar", + "off", + "--no-deps", + "--no-build-isolation", + "--verbose", + # "--verbose", ] - if verbose: - romp_args.append("--verbose") - - print(f"Fetching built wheels from log file: {remote_build_log_file}") - print(f"Using command:", " ".join(romp_args)) - - call(romp_args, verbose=verbose) - - wheel_filenames = [] - - for art in os.listdir(os.getcwd()): - if not art.endswith("artifacts.tar.gz") or not os.path.getsize(art): - continue - - print(f" Processing artifact archive: {art}") - wheel_fns = extract_tar(art, dest_dir) - for wfn in wheel_fns: - print(f" Retrieved built wheel: {wfn}") - wheel_filenames.extend(wheel_fns) - return wheel_filenames - - -def get_romp_pyos_options( - python_versions=PYTHON_VERSIONS, - operating_systems=PLATFORMS_BY_OS, -): - """ - Return a list of CLI options for romp - For example: - >>> expected = ['--version', '3.6', '--version', '3.7', '--version', '3.8', - ... '--version', '3.9', '--version', '3.10', '--platform', 'linux', - ... '--platform', 'macos', '--platform', 'windows'] - >>> assert get_romp_pyos_options() == expected - """ - python_dot_versions = [get_python_dot_version(pv) for pv in sorted(set(python_versions))] - pyos_options = list( - itertools.chain.from_iterable(("--version", ver) for ver in python_dot_versions) - ) + if environment: + eopts = environment.get_pip_cli_options() + cli_args.extend(eopts) + else: + print("WARNING: no download environment provided.") - pyos_options += list( - itertools.chain.from_iterable( - ("--platform", plat) for plat in sorted(set(operating_systems)) - ) - ) + cli_args.extend(requirements_specifiers) + for req_file in requirements_files: + cli_args.extend(["--requirement", req_file]) - return pyos_options + if TRACE: + print(f"Downloading wheels using command:", " ".join(cli_args)) + existing = set(os.listdir(dest_dir)) + error = False + try: + returncode, _stdout, stderr = call(cli_args, verbose=True) + if returncode != 0: + error = stderr + except Exception as e: + error = str(e) -def check_romp_is_configured(): - # these environment variable must be set before - has_envt = ( - os.environ.get("ROMP_BUILD_REQUEST_URL") - and os.environ.get("ROMP_DEFINITION_ID") - and os.environ.get("ROMP_PERSONAL_ACCESS_TOKEN") - and os.environ.get("ROMP_USERNAME") - ) + if error: + print() + print("###########################################################################") + print("##################### Failed to fetch all wheels ##########################") + print("###########################################################################") + print(error) + print() + print("###########################################################################") - if not has_envt: - raise Exception( - "ROMP_BUILD_REQUEST_URL, ROMP_DEFINITION_ID, " - "ROMP_PERSONAL_ACCESS_TOKEN and ROMP_USERNAME " - "are required enironment variables." - ) + downloaded = existing ^ set(os.listdir(dest_dir)) + return sorted(downloaded), error def build_wheels_locally_if_pure_python( @@ -3034,9 +2311,9 @@ def build_wheels_locally_if_pure_python( "--wheel-dir", wheel_dir, ] - + deps - + verbose - + [requirements_specifier] + +deps + +verbose + +[requirements_specifier] ) print(f"Building local wheels for: {requirements_specifier}") @@ -3064,95 +2341,6 @@ def build_wheels_locally_if_pure_python( return all_pure, pure_built -# TODO: Use me -def optimize_wheel(wheel_filename, dest_dir=THIRDPARTY_DIR): - """ - Optimize a wheel named `wheel_filename` in `dest_dir` such as renaming its - tags for PyPI compatibility and making it smaller if possible. Return the - name of the new wheel if renamed or the existing new name otherwise. - """ - if is_pure_wheel(wheel_filename): - print(f"Pure wheel: {wheel_filename}, nothing to do.") - return wheel_filename - - original_wheel_loc = os.path.join(dest_dir, wheel_filename) - wheel_dir = tempfile.mkdtemp(prefix="scancode-release-wheels-") - awargs = ["auditwheel", "addtag", "--wheel-dir", wheel_dir, original_wheel_loc] - call(awargs) - - audited = os.listdir(wheel_dir) - if not audited: - # cannot optimize wheel - return wheel_filename - - assert len(audited) == 1 - new_wheel_name = audited[0] - - new_wheel_loc = os.path.join(wheel_dir, new_wheel_name) - - # this needs to go now - os.remove(original_wheel_loc) - - if new_wheel_name == wheel_filename: - os.rename(new_wheel_loc, original_wheel_loc) - return wheel_filename - - new_wheel = Wheel.from_filename(new_wheel_name) - non_pypi_plats = utils_pypi_supported_tags.validate_platforms_for_pypi(new_wheel.platforms) - new_wheel.platforms = [p for p in new_wheel.platforms if p not in non_pypi_plats] - if not new_wheel.platforms: - print(f"Cannot make wheel PyPI compatible: {original_wheel_loc}") - os.rename(new_wheel_loc, original_wheel_loc) - return wheel_filename - - new_wheel_cleaned_filename = new_wheel.to_filename() - new_wheel_cleaned_loc = os.path.join(dest_dir, new_wheel_cleaned_filename) - os.rename(new_wheel_loc, new_wheel_cleaned_loc) - return new_wheel_cleaned_filename - - -def extract_tar( - location, - dest_dir=THIRDPARTY_DIR, -): - """ - Extract a tar archive at `location` in the `dest_dir` directory. Return a - list of extracted locations (either directories or files). - """ - with open(location, "rb") as fi: - with tarfile.open(fileobj=fi) as tar: - members = list(tar.getmembers()) - tar.extractall(dest_dir, members=members) - - return [os.path.basename(ti.name) for ti in members if ti.type == tarfile.REGTYPE] - - -def fetch_package_wheel(name, version, environment, dest_dir=THIRDPARTY_DIR): - """ - Fetch the binary wheel for package `name` and `version` and save in - `dest_dir`. Use the provided `environment` Environment to determine which - specific wheel to fetch. - - Return the fetched wheel file name on success or None if it was not fetched. - Trying fetching from our own remote repo, then from PyPI. - """ - wheel_filename = None - remote_package = get_remote_package(name=name, version=version) - if TRACE: - print(" remote_package:", remote_package) - if remote_package: - wheel_filename = remote_package.fetch_wheel(environment=environment, dest_dir=dest_dir) - if wheel_filename: - return wheel_filename - - pypi_package = get_pypi_package(name=name, version=version) - if TRACE: - print(" pypi_package:", pypi_package) - if pypi_package: - wheel_filename = pypi_package.fetch_wheel(environment=environment, dest_dir=dest_dir) - return wheel_filename - - def check_about(dest_dir=THIRDPARTY_DIR): try: subprocess.check_output(f"about check {dest_dir}".split()) @@ -3195,6 +2383,9 @@ def find_problems( def compute_normalized_license_expression(declared_licenses): + """ + Return a normalized license expression or None. + """ if not declared_licenses: return try: From 931f610aa8fd93aac6178b1f0beb3d55d4119ba0 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 5 Mar 2022 07:42:35 +0100 Subject: [PATCH 14/41] Cleanup whitespaces Signed-off-by: Philippe Ombredanne --- configure | 5 ++--- configure.bat | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/configure b/configure index c1d36aa..b2d64c4 100755 --- a/configure +++ b/configure @@ -55,7 +55,7 @@ CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin # Thirdparty package locations and index handling # Find packages from the local thirdparty directory or from thirdparty.aboutcode.org if [ -f "$CFG_ROOT_DIR/thirdparty" ]; then - PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty " + PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty" fi PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS --find-links https://thirdparty.aboutcode.org/pypi/simple/links.html" @@ -87,7 +87,7 @@ main() { done PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS $NO_INDEX" - + find_python create_virtualenv "$VIRTUALENV_DIR" install_packages "$CFG_REQUIREMENTS" @@ -197,7 +197,6 @@ clean() { } - main set +e diff --git a/configure.bat b/configure.bat index 961e0d9..2ae4727 100644 --- a/configure.bat +++ b/configure.bat @@ -209,4 +209,4 @@ for %%F in (%CLEANABLE%) do ( rmdir /s /q "%CFG_ROOT_DIR%\%%F" >nul 2>&1 del /f /q "%CFG_ROOT_DIR%\%%F" >nul 2>&1 ) -exit /b 0 \ No newline at end of file +exit /b 0 From 6e43a7a2a98322fc3da3ed61826757481b831c50 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 8 Mar 2022 16:17:33 -0800 Subject: [PATCH 15/41] Add usage instructions to README.rst Signed-off-by: Jono Yang --- README.rst | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 4173689..26bcdbc 100644 --- a/README.rst +++ b/README.rst @@ -9,7 +9,32 @@ our existing ones as well. Usage ===== -Usage instructions can be found in ``docs/skeleton-usage.rst``. + +A brand new project +------------------- +.. code-block:: bash + + git init my-new-repo + cd my-new-repo + git pull git@github.com:nexB/skeleton + + # Create the new repo on GitHub, then update your remote + git remote set-url origin git@github.com:nexB/your-new-repo.git + +From here, you can make the appropriate changes to the files for your specific project. + +Update an existing project +--------------------------- +.. code-block:: bash + + cd my-existing-project + git remote add skeleton git@github.com:nexB/skeleton + git fetch skeleton + git merge skeleton/main --allow-unrelated-histories + +This is also the workflow to use when updating the skeleton files in any given repository. + +More usage instructions can be found in ``docs/skeleton-usage.rst``. Release Notes ============= From b272e3b7c7e47a3143e0886ebc9e88b12c1c6eab Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 9 Mar 2022 14:08:43 +0100 Subject: [PATCH 16/41] Format code Signed-off-by: Philippe Ombredanne --- etc/scripts/utils_thirdparty.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index e303053..829cf8c 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -330,6 +330,7 @@ def get_package_versions( except RemoteNotFetchedException as e: print(f"Failed to fetch PyPI package {name} @ {version} info from {index_url}: {e}") + ################################################################################ # # Core models @@ -1617,6 +1618,7 @@ def tags(self): ) ) + ################################################################################ # # PyPI repo and link index for package wheels and sources @@ -1801,6 +1803,7 @@ def from_url(cls, url=ABOUT_BASE_URL, _LINKS_REPO={}): _LINKS_REPO[url] = cls(url=url) return _LINKS_REPO[url] + ################################################################################ # Globals for remote repos to be lazily created and cached on first use for the # life of the session together with some convenience functions. @@ -1834,6 +1837,7 @@ def get_pypi_package(name, version, index_url, verbose=TRACE_DEEP): except RemoteNotFetchedException as e: print(f"Failed to fetch PyPI package {name} @ {version} info from {index_url}: {e}") + ################################################################################ # # Basic file and URL-based operations using a persistent file-based Cache @@ -1994,6 +1998,7 @@ def fetch_and_save_path_or_url( fo.write(content) return content + ################################################################################ # Requirements processing ################################################################################ @@ -2031,6 +2036,7 @@ def get_required_packages( print(" get_required_packages: name:", name, "version:", version) yield repo.get_package(name, version) + ################################################################################ # Functions to update or fetch ABOUT and license files ################################################################################ @@ -2181,6 +2187,7 @@ def get_other_dists(_package, _dist): lic_errs = "\n".join(lic_errs) print(f"Failed to fetch some licenses:: {lic_errs}") + ################################################################################ # # Functions to build new Python wheels including native on multiple OSes @@ -2311,9 +2318,9 @@ def build_wheels_locally_if_pure_python( "--wheel-dir", wheel_dir, ] - +deps - +verbose - +[requirements_specifier] + + deps + + verbose + + [requirements_specifier] ) print(f"Building local wheels for: {requirements_specifier}") From 1e4d3bce4626494bb1392a063360e236caf77294 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 9 Mar 2022 16:56:19 +0100 Subject: [PATCH 17/41] Reorg setup sections This is now organized with more important data first. Signed-off-by: Philippe Ombredanne --- setup.cfg | 42 +++++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/setup.cfg b/setup.cfg index 81f762a..d8a7941 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,18 +1,15 @@ [metadata] -license_files = - apache-2.0.LICENSE - NOTICE - AUTHORS.rst - CHANGELOG.rst name = skeleton -author = nexB. Inc. and others -author_email = info@aboutcode.org license = Apache-2.0 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390 description = skeleton long_description = file:README.rst url = https://github.com/nexB/skeleton + +author = nexB. Inc. and others +author_email = info@aboutcode.org + classifiers = Development Status :: 5 - Production/Stable Intended Audience :: Developers @@ -20,27 +17,42 @@ classifiers = Programming Language :: Python :: 3 :: Only Topic :: Software Development Topic :: Utilities + keywords = utilities +license_files = + apache-2.0.LICENSE + NOTICE + AUTHORS.rst + CHANGELOG.rst + [options] -package_dir= +package_dir = =src -packages=find: +packages = find: include_package_data = true zip_safe = false -install_requires = + setup_requires = setuptools_scm[toml] >= 4 +python_requires = >=3.6.* + +install_requires = + + [options.packages.find] -where=src +where = src + [options.extras_require] testing = pytest >= 6, != 7.0.0 pytest-xdist >= 2 + aboutcode-toolkit >= 6.0.0 black -docs= - Sphinx>=3.3.1 - sphinx-rtd-theme>=0.5.0 - doc8>=0.8.1 + +docs = + Sphinx >= 3.3.1 + sphinx-rtd-theme >= 0.5.0 + doc8 >= 0.8.1 From 03d4799ac44a4def7dba1eb3a0ef3a280c663e43 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 9 Mar 2022 20:39:36 +0100 Subject: [PATCH 18/41] Do not depend on click. Use argparse. These boostrap scripts cannot depend on click. Signed-off-by: Philippe Ombredanne --- docs/skeleton-usage.rst | 16 +++--- etc/scripts/gen_requirements.py | 59 +++++++++++++--------- etc/scripts/gen_requirements_dev.py | 78 ++++++++++++++++------------- 3 files changed, 86 insertions(+), 67 deletions(-) diff --git a/docs/skeleton-usage.rst b/docs/skeleton-usage.rst index 7d16259..113bc71 100644 --- a/docs/skeleton-usage.rst +++ b/docs/skeleton-usage.rst @@ -49,7 +49,7 @@ customizing the skeleton files to your project: .. code-block:: bash - ./configure --init + ./configure This will initialize the virtual environment for the project, pull in the dependencies from PyPI and add them to the virtual environment. @@ -77,7 +77,7 @@ Replace \ with the version number of the Python being used, for exampl To generate requirements-dev.txt after requirements.txt has been generated: .. code-block:: bash - ./configure --init --dev + ./configure --dev python etc/scripts/gen_requirements_dev.py -s venv/lib/python/site-packages/ Note: on Windows, the ``site-packages`` directory is located at ``venv\Lib\site-packages\`` @@ -88,10 +88,11 @@ Note: on Windows, the ``site-packages`` directory is located at ``venv\Lib\site- .\configure --init --dev python .\\etc\\scripts\\gen_requirements_dev.py -s .\\venv\\Lib\\site-packages\\ + Collecting and generating ABOUT files for dependencies ------------------------------------------------------ -Ensure that the dependencies used by ``etc/scripts/bootstrap.py`` are installed: +Ensure that the dependencies used by ``etc/scripts/fetch_thirdparty.py`` are installed: .. code-block:: bash @@ -102,7 +103,7 @@ dependencies as wheels and generate ABOUT files for them: .. code-block:: bash - python etc/scripts/bootstrap.py -r requirements.txt -r requirements-dev.txt --with-deps + python etc/scripts/fetch_thirdparty.py -r requirements.txt -r requirements-dev.txt There may be issues with the generated ABOUT files, which will have to be corrected. You can check to see if your corrections are valid by running: @@ -122,8 +123,8 @@ Usage after project initialization Once the ``requirements.txt`` and ``requirements-dev.txt`` have been generated and the project dependencies and their ABOUT files have been uploaded to -thirdparty.aboutcode.org/pypi, you can configure the project without using the -``--init`` option. +thirdparty.aboutcode.org/pypi, you can configure the project as needed, typically +when you update dependencies or use a new checkout. If the virtual env for the project becomes polluted, or you would like to remove it, use the ``--clean`` option: @@ -146,12 +147,11 @@ update the dependencies in ``setup.cfg``, then run: .. code-block:: bash ./configure --clean # Remove existing virtual environment - ./configure --init # Create project virtual environment, pull in new dependencies source venv/bin/activate # Ensure virtual environment is activated python etc/scripts/gen_requirements.py -s venv/lib/python/site-packages/ # Regenerate requirements.txt python etc/scripts/gen_requirements_dev.py -s venv/lib/python/site-packages/ # Regenerate requirements-dev.txt pip install -r etc/scripts/requirements.txt # Install dependencies needed by etc/scripts/bootstrap.py - python etc/scripts/bootstrap.py -r requirements.txt -r requirements-dev.txt --with-deps # Collect dependency wheels and their ABOUT files + python etc/scripts/fetch_thirdparty.py -r requirements.txt -r requirements-dev.txt # Collect dependency wheels and their ABOUT files Ensure that the generated ABOUT files are valid, then take the dependency wheels and ABOUT files and upload them to thirdparty.aboutcode.org/pypi. diff --git a/etc/scripts/gen_requirements.py b/etc/scripts/gen_requirements.py index 6f17a75..07e26f7 100644 --- a/etc/scripts/gen_requirements.py +++ b/etc/scripts/gen_requirements.py @@ -8,37 +8,48 @@ # See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # -import click +import argparse +import pathlib + import utils_requirements +""" +Utilities to manage requirements files. +NOTE: this should use ONLY the standard library and not import anything else +because this is used for boostrapping with no requirements installed. +""" -@click.command() -@click.option( - "-s", - "--site-packages-dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False, resolve_path=True), - required=True, - metavar="DIR", - help='Path to the "site-packages" directory where wheels are installed such as lib/python3.6/site-packages', -) -@click.option( - "-r", - "--requirements-file", - type=click.Path(path_type=str, dir_okay=False), - metavar="FILE", - default="requirements.txt", - show_default=True, - help="Path to the requirements file to update or create.", -) -@click.help_option("-h", "--help") -def gen_requirements(site_packages_dir, requirements_file): - """ + +def gen_requirements(): + description = """ Create or replace the `--requirements-file` file FILE requirements file with all locally installed Python packages.all Python packages found installed in `--site-packages-dir` """ + parser = argparse.ArgumentParser(description=description) + + parser.add_argument( + "-s", + "--site-packages-dir", + dest="site_packages_dir", + type=pathlib.Path, + required=True, + metavar="DIR", + help="Path to the 'site-packages' directory where wheels are installed such as lib/python3.6/site-packages", + ) + parser.add_argument( + "-r", + "--requirements-file", + type=pathlib.Path, + metavar="FILE", + default="requirements.txt", + help="Path to the requirements file to update or create.", + ) + + args = parser.parse_args() + utils_requirements.lock_requirements( - requirements_file=requirements_file, - site_packages_dir=site_packages_dir, + site_packages_dir=args.site_packages_dir, + requirements_file=args.requirements_file, ) diff --git a/etc/scripts/gen_requirements_dev.py b/etc/scripts/gen_requirements_dev.py index ef80455..12cc06d 100644 --- a/etc/scripts/gen_requirements_dev.py +++ b/etc/scripts/gen_requirements_dev.py @@ -8,51 +8,59 @@ # See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # -import click +import argparse +import pathlib + import utils_requirements +""" +Utilities to manage requirements files. +NOTE: this should use ONLY the standard library and not import anything else +because this is used for boostrapping with no requirements installed. +""" -@click.command() -@click.option( - "-s", - "--site-packages-dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False, resolve_path=True), - required=True, - metavar="DIR", - help='Path to the "site-packages" directory where wheels are installed such as lib/python3.6/site-packages', -) -@click.option( - "-d", - "--dev-requirements-file", - type=click.Path(path_type=str, dir_okay=False), - metavar="FILE", - default="requirements-dev.txt", - show_default=True, - help="Path to the dev requirements file to update or create.", -) -@click.option( - "-r", - "--main-requirements-file", - type=click.Path(path_type=str, dir_okay=False), - default="requirements.txt", - metavar="FILE", - show_default=True, - help="Path to the main requirements file. Its requirements will be excluded " - "from the generated dev requirements.", -) -@click.help_option("-h", "--help") -def gen_dev_requirements(site_packages_dir, dev_requirements_file, main_requirements_file): - """ + +def gen_dev_requirements(): + description = """ Create or overwrite the `--dev-requirements-file` pip requirements FILE with all Python packages found installed in `--site-packages-dir`. Exclude package names also listed in the --main-requirements-file pip requirements FILE (that are assume to the production requirements and therefore to always be present in addition to the development requirements). """ + parser = argparse.ArgumentParser(description=description) + + parser.add_argument( + "-s", + "--site-packages-dir", + type=pathlib.Path, + required=True, + metavar="DIR", + help='Path to the "site-packages" directory where wheels are installed such as lib/python3.6/site-packages', + ) + parser.add_argument( + "-d", + "--dev-requirements-file", + type=pathlib.Path, + metavar="FILE", + default="requirements-dev.txt", + help="Path to the dev requirements file to update or create.", + ) + parser.add_argument( + "-r", + "--main-requirements-file", + type=pathlib.Path, + default="requirements.txt", + metavar="FILE", + help="Path to the main requirements file. Its requirements will be excluded " + "from the generated dev requirements.", + ) + args = parser.parse_args() + utils_requirements.lock_dev_requirements( - dev_requirements_file=dev_requirements_file, - main_requirements_file=main_requirements_file, - site_packages_dir=site_packages_dir, + dev_requirements_file=args.dev_requirements_file, + main_requirements_file=args.main_requirements_file, + site_packages_dir=args.site_packages_dir, ) From f0d5a2979c9e04c7f77c5cf76aea5c936ee31ac3 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 9 Mar 2022 20:47:06 +0100 Subject: [PATCH 19/41] Correct configure scripts These were no longer working Signed-off-by: Philippe Ombredanne --- configure | 53 +++++++++++++++++++++++---------------------------- configure.bat | 5 +---- 2 files changed, 25 insertions(+), 33 deletions(-) diff --git a/configure b/configure index b2d64c4..93a836b 100755 --- a/configure +++ b/configure @@ -67,34 +67,6 @@ if [[ "$CFG_QUIET" == "" ]]; then fi -################################ -# Main command line entry point -main() { - CFG_REQUIREMENTS=$REQUIREMENTS - NO_INDEX="--no-index" - - # We are using getopts to parse option arguments that start with "-" - while getopts :-: optchar; do - case "${optchar}" in - -) - case "${OPTARG}" in - help ) cli_help;; - clean ) find_python && clean;; - dev ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS";; - init ) NO_INDEX="";; - esac;; - esac - done - - PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS $NO_INDEX" - - find_python - create_virtualenv "$VIRTUALENV_DIR" - install_packages "$CFG_REQUIREMENTS" - . "$CFG_BIN_DIR/activate" -} - - ################################ # Find a proper Python to run # Use environment variables or a file if available. @@ -197,6 +169,29 @@ clean() { } -main +################################ +# Main command line entry point +CFG_REQUIREMENTS=$REQUIREMENTS + +# We are using getopts to parse option arguments that start with "-" +while getopts :-: optchar; do + case "${optchar}" in + -) + case "${OPTARG}" in + help ) cli_help;; + clean ) find_python && clean;; + dev ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS";; + init ) ;; + esac;; + esac +done + +PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS" + +find_python +create_virtualenv "$VIRTUALENV_DIR" +install_packages "$CFG_REQUIREMENTS" +. "$CFG_BIN_DIR/activate" + set +e diff --git a/configure.bat b/configure.bat index 2ae4727..7001514 100644 --- a/configure.bat +++ b/configure.bat @@ -77,14 +77,11 @@ if not "%1" == "" ( if "%1" EQU "--dev" ( set "CFG_REQUIREMENTS=%DEV_REQUIREMENTS%" ) - if "%1" EQU "--init" ( - set "NO_INDEX= " - ) shift goto again ) -set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% %NO_INDEX%" +set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS%" @rem ################################ From 6ed9983e882b195b3093434f70fd0d0f01d8399f Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 9 Mar 2022 20:50:04 +0100 Subject: [PATCH 20/41] Remove remnants of configure --init This does not exists anymore Signed-off-by: Philippe Ombredanne --- configure | 5 ----- configure.bat | 4 ---- docs/skeleton-usage.rst | 3 ++- 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/configure b/configure index 93a836b..8c5f4ab 100755 --- a/configure +++ b/configure @@ -137,14 +137,10 @@ cli_help() { echo " usage: ./configure [options]" echo echo The default is to configure for regular use. Use --dev for development. - echo Use the --init option if starting a new project and the project - echo dependencies are not available on thirdparty.aboutcode.org/pypi/ - echo and requirements.txt and/or requirements-dev.txt has not been generated. echo echo The options are: echo " --clean: clean built and installed files and exit." echo " --dev: configure the environment for development." - echo " --init: pull dependencies from PyPI. Used when first setting up a project." echo " --help: display this help message and exit." echo echo By default, the python interpreter version found in the path is used. @@ -181,7 +177,6 @@ while getopts :-: optchar; do help ) cli_help;; clean ) find_python && clean;; dev ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS";; - init ) ;; esac;; esac done diff --git a/configure.bat b/configure.bat index 7001514..e38b5fb 100644 --- a/configure.bat +++ b/configure.bat @@ -180,14 +180,10 @@ exit /b 0 echo " usage: configure [options]" echo " " echo The default is to configure for regular use. Use --dev for development. - echo Use the --init option if starting a new project and the project - echo dependencies are not available on thirdparty.aboutcode.org/pypi/ - echo and requirements.txt and/or requirements-dev.txt has not been generated. echo " " echo The options are: echo " --clean: clean built and installed files and exit." echo " --dev: configure the environment for development." - echo " --init: pull dependencies from PyPI. Used when first setting up a project." echo " --help: display this help message and exit." echo " " echo By default, the python interpreter version found in the path is used. diff --git a/docs/skeleton-usage.rst b/docs/skeleton-usage.rst index 113bc71..ad9b9ff 100644 --- a/docs/skeleton-usage.rst +++ b/docs/skeleton-usage.rst @@ -54,6 +54,7 @@ customizing the skeleton files to your project: This will initialize the virtual environment for the project, pull in the dependencies from PyPI and add them to the virtual environment. + Generating requirements.txt and requirements-dev.txt ---------------------------------------------------- @@ -85,7 +86,7 @@ Note: on Windows, the ``site-packages`` directory is located at ``venv\Lib\site- .. code-block:: bash python .\\etc\\scripts\\gen_requirements.py -s .\\venv\\Lib\\site-packages\\ - .\configure --init --dev + .\configure --dev python .\\etc\\scripts\\gen_requirements_dev.py -s .\\venv\\Lib\\site-packages\\ From bf6bbaac67cb8fa31d205b2d76e538a3bed8780f Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 11 Mar 2022 07:52:07 +0100 Subject: [PATCH 21/41] Pytyon 3.6 is not available on Windows 2022 Signed-off-by: Philippe Ombredanne --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 089abe9..6ca19c4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -35,7 +35,7 @@ jobs: parameters: job_name: macos11_cpython image_name: macos-11 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.7', '3.8', '3.9', '3.10'] test_suites: all: venv/bin/pytest -n 2 -vvs From 4ab834f15860a22bcbca2a5ea567ce5f39c7c345 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 11 Mar 2022 09:42:46 +0100 Subject: [PATCH 22/41] Add long_description_content_type Twine and PyPI prefer having it. Signed-off-by: Philippe Ombredanne --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index d8a7941..12d6654 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,6 +5,7 @@ license = Apache-2.0 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390 description = skeleton long_description = file:README.rst +long_description_content_type = text/x-rst url = https://github.com/nexB/skeleton author = nexB. Inc. and others From feea014129f87367b6358c7f322a3d824174bde7 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 12 Mar 2022 18:48:21 +0100 Subject: [PATCH 23/41] Streamline string representation of deps Use fstrings and ensure they are "idem potent" Signed-off-by: Philippe Ombredanne --- src/debian_inspector/deps.py | 11 ++++------- tests/test_deps.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/src/debian_inspector/deps.py b/src/debian_inspector/deps.py index 9e5228b..bde0ecc 100644 --- a/src/debian_inspector/deps.py +++ b/src/debian_inspector/deps.py @@ -1,12 +1,12 @@ # # Copyright (c) nexB Inc. and others. All rights reserved. +# Copyright (c) 2018 Peter Odding +# Author: Peter Odding +# URL: https://github.com/xolox/python-deb-pkg-tools # SPDX-License-Identifier: Apache-2.0 AND MIT # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. # See https://github.com/nexB/debian-inspector for support or download. # See https://aboutcode.org for more information about nexB OSS projects. -# Copyright (c) 2018 Peter Odding -# Author: Peter Odding -# URL: https://github.com/xolox/python-deb-pkg-tools import re @@ -227,12 +227,9 @@ def matches(self, name, version=None, architecture=None): return None def __str__(self, *args, **kwargs): - - s = '{name} ({operator} {version})'.format( - name=self.name, operator=self.operator, version=self.version) + s = f'{self.name} ({self.operator} {self.version})' if self.architectures: s += ' [{}]'.format(' '.join(self.architectures)) - return s diff --git a/tests/test_deps.py b/tests/test_deps.py index 6e916d0..5669b31 100644 --- a/tests/test_deps.py +++ b/tests/test_deps.py @@ -140,3 +140,36 @@ def test_parse_relationship(self): assert rel == deps.Relationship(name='python') rel = deps.parse_relationship('python (<< 3)') assert rel == deps.VersionedRelationship(name='python', operator='<<', version='3') + + def test_parse_depends_complex_cases_is_idempotent(self): + depends = [ + 'clamav-freshclam (>= 0.103.2+dfsg) | clamav-data, libc6 (>= 2.15), libclamav9 (>= 0.103.2), ' + 'libcurl3 (>= 7.16.2), libjson-c2 (>= 0.10), libssl1.0.0 (>= 1.0.0), zlib1g (>= 1:1.2.3.3)', + 'sysvinit-utils (>= 2.86.ds1-62), insserv (>> 1.12.0-10)', + 'libgimp2.0 (>= 2.8.16), libgimp2.0 (<= 2.8.16-z)', + 'zlib1g (>= 1:1.1.4), python:any (>= 2.6.6-7~)', + 'git (>> 1:2.34.1), git (<< 1:2.34.1-.)', + 'libnspr4 (>= 2:4.13.1), libnspr4 (<= 2:4.13.1-0ubuntu0.16.04.1.1~)', + 'libc6 (>> 2.23), libc6 (<< 2.24)', + 'libptexenc1 (>= 2015.20160222.37495-1ubuntu0.1), libptexenc1 (<< 2015.20160222.37495-1ubuntu0.1.1~), ' + 'libkpathsea6 (>= 2015.20160222.37495-1ubuntu0.1), libkpathsea6 (<< 2015.20160222.37495-1ubuntu0.1.1~), ' + 'libsynctex1 (>= 2015.20160222.37495-1ubuntu0.1), libsynctex1 (<< 2015.20160222.37495-1ubuntu0.1.1~), ' + 'libtexlua52 (>= 2015.20160222.37495-1ubuntu0.1), libtexlua52 (<< 2015.20160222.37495-1ubuntu0.1.1~), ' + 'libtexluajit2 (>= 2015.20160222.37495-1ubuntu0.1), libtexluajit2 (<< 2015.20160222.37495-1ubuntu0.1.1~)', + 'libldb1 (<< 2:1.1.25~), libldb1 (>> 2:1.1.24~)', + 'libnettle6 (= 3.2-1ubuntu0.16.04.2), libhogweed4 (= 3.2-1ubuntu0.16.04.2), libgmp10-dev, dpkg (>= 1.15.4) | install-info', + 'libkf5widgetsaddons-data (= 5.18.0-0ubuntu1), libc6 (>= 2.14), libqt5core5a (>= 5.5.0), ' + 'libqt5gui5 (>= 5.3.0) | libqt5gui5-gles (>= 5.3.0), libqt5widgets5 (>= 5.2.0), libstdc++6 (>= 4.1.1)', + 'libnuma1 (= 2.0.11-1ubuntu1.1), libc6-dev | libc-dev', + 'perl, perl (>= 5.11.2) | libextutils-cbuilder-perl, perl (>= 5.12) | libextutils-parsexs-perl, ' + 'perl (>= 5.13.9) | libmodule-metadata-perl, perl (>= 5.13.9) | libversion-perl (>= 1:0.8700), ' + 'perl (>= 5.19.5) | libtest-harness-perl (>= 3.29), perl (>= 5.21.3) | libcpan-meta-perl (>= 2.142060)', + 'libc6 (>= 2.14), libglib2.0-0 (>= 2.37.3), libgtk-3-0 (>= 3.11.5), libgtksourceview-3.0-1 (>= 3.17.3), ' + 'libpeas-1.0-0 (>= 1.0.0), libzeitgeist-2.0-0 (>= 0.9.9), dconf-gsettings-backend | gsettings-backend, ' + 'python3 (<< 3.6), python3 (>= 3.5~), python3.5, gedit (>= 3.18), gedit (<< 3.19), gir1.2-git2-glib-1.0, ' + 'gir1.2-glib-2.0, gir1.2-gtk-3.0, gir1.2-gtksource-3.0, gir1.2-gucharmap-2.90, gir1.2-pango-1.0, gir1.2-peas-1.0, ' + 'gir1.2-vte-2.91, gir1.2-zeitgeist-2.0, python3-gi, python3-gi-cairo, python3-cairo, python3-dbus', + ] + + for depend in depends: + assert str(deps.parse_depends(depend)) == depend From 4ef463fdbbcc1c108307b07e26b4a231d2229799 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 14 Mar 2022 11:12:54 +0100 Subject: [PATCH 24/41] Run fewer Azure jobs This new configuration means that all the Python versions are tested in a single CI job. This allows doing fewer checkouts and improves CI throughput overall. Signed-off-by: Philippe Ombredanne --- etc/ci/azure-posix.yml | 31 ++++++++++++++----------------- etc/ci/azure-win.yml | 30 +++++++++++++++--------------- 2 files changed, 29 insertions(+), 32 deletions(-) diff --git a/etc/ci/azure-posix.yml b/etc/ci/azure-posix.yml index 7a9acff..9fdc7f1 100644 --- a/etc/ci/azure-posix.yml +++ b/etc/ci/azure-posix.yml @@ -13,10 +13,8 @@ jobs: strategy: matrix: - ${{ each pyver in parameters.python_versions }}: ${{ each tsuite in parameters.test_suites }}: - ${{ format('py{0} {1}', pyver, tsuite.key) }}: - python_version: ${{ pyver }} + ${{ tsuite.key }}: test_suite_label: ${{ tsuite.key }} test_suite: ${{ tsuite.value }} @@ -24,19 +22,18 @@ jobs: - checkout: self fetchDepth: 10 - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python_version)' - architecture: '${{ parameters.python_architecture }}' - displayName: 'Install Python $(python_version)' + - ${{ each pyver in parameters.python_versions }}: + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ pyver }}' + architecture: '${{ parameters.python_architecture }}' + displayName: '${{ pyver }} - Install Python' - - script: | - python --version - python3 --version - python$(python_version) --version - echo "python$(python_version)" > PYTHON_EXECUTABLE - ./configure --dev - displayName: 'Run Configure' + - script: | + python${{ pyver }} --version + echo "python${{ pyver }}" > PYTHON_EXECUTABLE + ./configure --clean && ./configure --dev + displayName: '${{ pyver }} - Configure' - - script: $(test_suite) - displayName: 'Run $(test_suite_label) tests with py$(python_version) on ${{ parameters.job_name }}' + - script: $(test_suite) + displayName: '${{ pyver }} - $(test_suite_label) on ${{ parameters.job_name }}' diff --git a/etc/ci/azure-win.yml b/etc/ci/azure-win.yml index 03d8927..26b4111 100644 --- a/etc/ci/azure-win.yml +++ b/etc/ci/azure-win.yml @@ -13,27 +13,27 @@ jobs: strategy: matrix: - ${{ each pyver in parameters.python_versions }}: ${{ each tsuite in parameters.test_suites }}: - ${{ format('py{0} {1}', pyver, tsuite.key) }}: - python_version: ${{ pyver }} + ${{ tsuite.key }}: test_suite_label: ${{ tsuite.key }} test_suite: ${{ tsuite.value }} + steps: - checkout: self fetchDepth: 10 - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python_version)' - architecture: '${{ parameters.python_architecture }}' - displayName: 'Install Python $(python_version)' + - ${{ each pyver in parameters.python_versions }}: + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ pyver }}' + architecture: '${{ parameters.python_architecture }}' + displayName: '${{ pyver }} - Install Python' - - script: | - python --version - echo | set /p=python> PYTHON_EXECUTABLE - configure --dev - displayName: 'Run Configure' + - script: | + python --version + echo | set /p=python> PYTHON_EXECUTABLE + configure --clean && configure --dev + displayName: '${{ pyver }} - Configure' - - script: $(test_suite) - displayName: 'Run $(test_suite_label) tests with py$(python_version) on ${{ parameters.job_name }}' + - script: $(test_suite) + displayName: '${{ pyver }} - $(test_suite_label) on ${{ parameters.job_name }}' From e9210529fbe09a498abf85d27154110a34728fcb Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Tue, 22 Mar 2022 21:13:44 +0530 Subject: [PATCH 25/41] Add RTD css templates Adds changes to conf.py and html template theme_overrides.css created by @johnmhoran Signed-off-by: Ayan Sinha Mahapatra --- docs/source/_static/theme_overrides.css | 353 ++++++++++++++++++++++++ docs/source/conf.py | 21 ++ 2 files changed, 374 insertions(+) create mode 100644 docs/source/_static/theme_overrides.css diff --git a/docs/source/_static/theme_overrides.css b/docs/source/_static/theme_overrides.css new file mode 100644 index 0000000..9662d63 --- /dev/null +++ b/docs/source/_static/theme_overrides.css @@ -0,0 +1,353 @@ +body { + color: #000000; +} + +p { + margin-bottom: 10px; +} + +.wy-plain-list-disc, .rst-content .section ul, .rst-content .toctree-wrapper ul, article ul { + margin-bottom: 10px; +} + +.custom_header_01 { + color: #cc0000; + font-size: 22px; + font-weight: bold; + line-height: 50px; +} + +h1, h2, h3, h4, h5, h6 { + margin-bottom: 20px; + margin-top: 20px; +} + +h5 { + font-size: 18px; + color: #000000; + font-style: italic; + margin-bottom: 10px; +} + +h6 { + font-size: 15px; + color: #000000; + font-style: italic; + margin-bottom: 10px; +} + +/* custom admonitions */ +/* success */ +.custom-admonition-success .admonition-title { + color: #000000; + background: #ccffcc; + border-radius: 5px 5px 0px 0px; +} +div.custom-admonition-success.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #cccccc; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + +/* important */ +.custom-admonition-important .admonition-title { + color: #000000; + background: #ccffcc; + border-radius: 5px 5px 0px 0px; + border-bottom: solid 1px #000000; +} +div.custom-admonition-important.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #cccccc; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + +/* caution */ +.custom-admonition-caution .admonition-title { + color: #000000; + background: #ffff99; + border-radius: 5px 5px 0px 0px; + border-bottom: solid 1px #e8e8e8; +} +div.custom-admonition-caution.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #cccccc; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + +/* note */ +.custom-admonition-note .admonition-title { + color: #ffffff; + background: #006bb3; + border-radius: 5px 5px 0px 0px; +} +div.custom-admonition-note.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #cccccc; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + +/* todo */ +.custom-admonition-todo .admonition-title { + color: #000000; + background: #cce6ff; + border-radius: 5px 5px 0px 0px; + border-bottom: solid 1px #99ccff; +} +div.custom-admonition-todo.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #99ccff; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + +/* examples */ +.custom-admonition-examples .admonition-title { + color: #000000; + background: #ffe6cc; + border-radius: 5px 5px 0px 0px; + border-bottom: solid 1px #d8d8d8; +} +div.custom-admonition-examples.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #cccccc; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + +.wy-nav-content { + max-width: 100%; + padding-right: 100px; + padding-left: 100px; + background-color: #f2f2f2; +} + +div.rst-content { + background-color: #ffffff; + border: solid 1px #e5e5e5; + padding: 20px 40px 20px 40px; +} + +.rst-content .guilabel { + border: 1px solid #ffff99; + background: #ffff99; + font-size: 100%; + font-weight: normal; + border-radius: 4px; + padding: 2px 0px; + margin: auto 2px; + vertical-align: middle; +} + +.rst-content kbd { + font-family: SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",Courier,monospace; + border: solid 1px #d8d8d8; + background-color: #f5f5f5; + padding: 0px 3px; + border-radius: 3px; +} + +.wy-nav-content-wrap a { + color: #0066cc; + text-decoration: none; +} +.wy-nav-content-wrap a:hover { + color: #0099cc; + text-decoration: underline; +} + +.wy-nav-top a { + color: #ffffff; +} + +/* Based on numerous similar approaches e.g., https://github.com/readthedocs/sphinx_rtd_theme/issues/117 and https://rackerlabs.github.io/docs-rackspace/tools/rtd-tables.html -- but remove form-factor limits to enable table wrap on full-size and smallest-size form factors */ +.wy-table-responsive table td { + white-space: normal !important; +} + +.rst-content table.docutils td, +.rst-content table.docutils th { + padding: 5px 10px 5px 10px; +} +.rst-content table.docutils td p, +.rst-content table.docutils th p { + font-size: 14px; + margin-bottom: 0px; +} +.rst-content table.docutils td p cite, +.rst-content table.docutils th p cite { + font-size: 14px; + background-color: transparent; +} + +.colwidths-given th { + border: solid 1px #d8d8d8 !important; +} +.colwidths-given td { + border: solid 1px #d8d8d8 !important; +} + +/*handles single-tick inline code*/ +.wy-body-for-nav cite { + color: #000000; + background-color: transparent; + font-style: normal; + font-family: "Courier New"; + font-size: 13px; + padding: 3px 3px 3px 3px; +} + +.rst-content pre.literal-block, .rst-content div[class^="highlight"] pre, .rst-content .linenodiv pre { + font-family: SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",Courier,monospace; + font-size: 13px; + overflow: visible; + white-space: pre-wrap; + color: #000000; +} + +.rst-content pre.literal-block, .rst-content div[class^='highlight'] { + background-color: #f8f8f8; + border: solid 1px #e8e8e8; +} + +/* This enables inline code to wrap. */ +code, .rst-content tt, .rst-content code { + white-space: pre-wrap; + padding: 2px 3px 1px; + border-radius: 3px; + font-size: 13px; + background-color: #ffffff; +} + +/* use this added class for code blocks attached to bulleted list items */ +.highlight-top-margin { + margin-top: 20px !important; +} + +/* change color of inline code block */ +span.pre { + color: #e01e5a; +} + +.wy-body-for-nav blockquote { + margin: 1em 0; + padding-left: 1em; + border-left: 4px solid #ddd; + color: #000000; +} + +/* Fix the unwanted top and bottom padding inside a nested bulleted/numbered list */ +.rst-content .section ol p, .rst-content .section ul p { + margin-bottom: 0px; +} + +/* add spacing between bullets for legibility */ +.rst-content .section ol li, .rst-content .section ul li { + margin-bottom: 5px; +} + +.rst-content .section ol li:first-child, .rst-content .section ul li:first-child { + margin-top: 5px; +} + +/* but exclude the toctree bullets */ +.rst-content .toctree-wrapper ul li, .rst-content .toctree-wrapper ul li:first-child { + margin-top: 0px; + margin-bottom: 0px; +} + +/* remove extra space at bottom of multine list-table cell */ +.rst-content .line-block { + margin-left: 0px; + margin-bottom: 0px; + line-height: 24px; +} + +/* fix extra vertical spacing in page toctree */ +.rst-content .toctree-wrapper ul li ul, article ul li ul { + margin-top: 0; + margin-bottom: 0; +} + +/* this is used by the genindex added via layout.html (see source/_templates/) to sidebar toc */ +.reference.internal.toc-index { + color: #d9d9d9; +} + +.reference.internal.toc-index.current { + background-color: #ffffff; + color: #000000; + font-weight: bold; +} + +.toc-index-div { + border-top: solid 1px #000000; + margin-top: 10px; + padding-top: 5px; +} + +.indextable ul li { + font-size: 14px; + margin-bottom: 5px; +} + +/* The next 2 fix the poor vertical spacing in genindex.html (the alphabetized index) */ +.indextable.genindextable { + margin-bottom: 20px; +} + +div.genindex-jumpbox { + margin-bottom: 10px; +} + +/* rst image classes */ + +.clear-both { + clear: both; + } + +.float-left { + float: left; + margin-right: 20px; +} + +img { + border: solid 1px #e8e8e8; +} + +/* These are custom and need to be defined in conf.py to access in all pages, e.g., '.. role:: red' */ +.img-title { + color: #000000; + /* neither padding nor margin works for vertical spacing bc it's a span -- line-height does, sort of */ + line-height: 3.0; + font-style: italic; + font-weight: 600; +} + +.img-title-para { + color: #000000; + margin-top: 20px; + margin-bottom: 0px; + font-style: italic; + font-weight: 500; +} + +.red { + color: red; +} diff --git a/docs/source/conf.py b/docs/source/conf.py index 74b8649..778636e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -74,3 +74,24 @@ "github_version": "develop", # branch "conf_py_path": "/docs/source/", # path in the checkout to the docs root } + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +html_show_sphinx = True + +# Define CSS and HTML abbreviations used in .rst files. These are examples. +# .. role:: is used to refer to styles defined in _static/theme_overrides.css and is used like this: :red:`text` +rst_prolog = """ +.. |psf| replace:: Python Software Foundation + +.. # define a hard line break for HTML +.. |br| raw:: html + +
+ +.. role:: red + +.. role:: img-title + +.. role:: img-title-para + +""" From bd2df2a9608fe02e5d725ab8d9b03a00fdb0ed7a Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 24 Mar 2022 15:12:18 +0530 Subject: [PATCH 26/41] Add GitHub action for doc build tests Signed-off-by: Ayan Sinha Mahapatra --- .github/workflows/docs-ci.yml | 37 ++++++++++++++++++++++++++++++++++ docs/source/skeleton/index.rst | 4 ++-- 2 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/docs-ci.yml diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml new file mode 100644 index 0000000..656d624 --- /dev/null +++ b/.github/workflows/docs-ci.yml @@ -0,0 +1,37 @@ +name: CI Documentation + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-20.04 + + strategy: + max-parallel: 4 + matrix: + python-version: [3.7] + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Give permission to run scripts + run: chmod +x ./docs/scripts/doc8_style_check.sh + + - name: Install Dependencies + run: pip install -e .[docs] + + - name: Check Sphinx Documentation build minimally + working-directory: ./docs + run: sphinx-build -E -W source build + + - name: Check for documentation style errors + working-directory: ./docs + run: ./scripts/doc8_style_check.sh + + diff --git a/docs/source/skeleton/index.rst b/docs/source/skeleton/index.rst index 7dfc6cb..f99cdec 100644 --- a/docs/source/skeleton/index.rst +++ b/docs/source/skeleton/index.rst @@ -2,14 +2,14 @@ # Rst docs - https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html # # 1. Place docs in folders under source for different sections -# 2. Link them by adding individual index files in each section +# 2. Link them by adding individual index files in each section # to the main index, and then files for each section to their # respective index files. # 3. Use `.. include` statements to include other .rst files # or part of them, or use hyperlinks to a section of the docs, # to get rid of repetition. # https://docutils.sourceforge.io/docs/ref/rst/directives.html#including-an-external-document-fragment -# +# # Note: Replace these guide/placeholder docs .. include:: ../../../README.rst From 3e2d801c69cc1c7523d1613bc9c3e3d805b85d3b Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 24 Mar 2022 15:59:39 +0530 Subject: [PATCH 27/41] Fix conf.py to fix doc build Signed-off-by: Ayan Sinha Mahapatra --- docs/source/conf.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 778636e..62bca04 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -65,9 +65,6 @@ master_doc = 'index' html_context = { - "css_files": [ - "_static/theme_overrides.css", # override wide tables in RTD theme - ], "display_github": True, "github_user": "nexB", "github_repo": "nexb-skeleton", @@ -75,6 +72,11 @@ "conf_py_path": "/docs/source/", # path in the checkout to the docs root } +html_css_files = [ + '_static/theme_overrides.css' + ] + + # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. html_show_sphinx = True From eb578898a0f48ca04f5071dbbfb460de35eb5383 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 24 Mar 2022 20:21:44 +0530 Subject: [PATCH 28/41] Add docs option in configure Adds a --docs option to the configure script to also install requirements for the documentation builds. Signed-off-by: Ayan Sinha Mahapatra --- configure | 2 ++ configure.bat | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/configure b/configure index 8c5f4ab..715c99f 100755 --- a/configure +++ b/configure @@ -30,6 +30,7 @@ CLI_ARGS=$1 # Requirement arguments passed to pip and used by default or with --dev. REQUIREMENTS="--editable . --constraint requirements.txt" DEV_REQUIREMENTS="--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt" +DOCS_REQUIREMENTS="--editable .[docs] --constraint requirements.txt" # where we create a virtualenv VIRTUALENV_DIR=venv @@ -177,6 +178,7 @@ while getopts :-: optchar; do help ) cli_help;; clean ) find_python && clean;; dev ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS";; + docs ) CFG_REQUIREMENTS="$DOCS_REQUIREMENTS";; esac;; esac done diff --git a/configure.bat b/configure.bat index e38b5fb..487e78a 100644 --- a/configure.bat +++ b/configure.bat @@ -28,6 +28,7 @@ @rem # Requirement arguments passed to pip and used by default or with --dev. set "REQUIREMENTS=--editable . --constraint requirements.txt" set "DEV_REQUIREMENTS=--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt" +set "DOCS_REQUIREMENTS=--editable .[docs] --constraint requirements.txt" @rem # where we create a virtualenv set "VIRTUALENV_DIR=venv" @@ -77,6 +78,9 @@ if not "%1" == "" ( if "%1" EQU "--dev" ( set "CFG_REQUIREMENTS=%DEV_REQUIREMENTS%" ) + if "%1" EQU "--docs" ( + set "CFG_REQUIREMENTS=%DOCS_REQUIREMENTS%" + ) shift goto again ) From 5556e71f0e3f780b4dd955e1f3b93395d345c36c Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 30 Mar 2022 15:28:39 +0530 Subject: [PATCH 29/41] Add documentation contribute page Adds documentation page on contributing to the docs, and also modifies directory structure to avoid having the skeleton directory in docs merged in projects. Signed-off-by: Ayan Sinha Mahapatra --- docs/source/contribute/contrib_doc.rst | 314 +++++++++++++++++++++++++ docs/source/index.rst | 3 +- docs/{ => source}/skeleton-usage.rst | 4 +- docs/source/skeleton/index.rst | 15 -- 4 files changed, 319 insertions(+), 17 deletions(-) create mode 100644 docs/source/contribute/contrib_doc.rst rename docs/{ => source}/skeleton-usage.rst (98%) delete mode 100644 docs/source/skeleton/index.rst diff --git a/docs/source/contribute/contrib_doc.rst b/docs/source/contribute/contrib_doc.rst new file mode 100644 index 0000000..13882e1 --- /dev/null +++ b/docs/source/contribute/contrib_doc.rst @@ -0,0 +1,314 @@ +.. _contrib_doc_dev: + +Contributing to the Documentation +================================= + +.. _contrib_doc_setup_local: + +Setup Local Build +----------------- + +To get started, create or identify a working directory on your local machine. + +Open that directory and execute the following command in a terminal session:: + + git clone https://github.com/nexB/skeleton.git + +That will create an ``/skeleton`` directory in your working directory. +Now you can install the dependencies in a virtualenv:: + + cd skeleton + ./configure --docs + +.. note:: + + In case of windows, run ``configure --docs`` instead of this. + +Now, this will install the following prerequisites: + +- Sphinx +- sphinx_rtd_theme (the format theme used by ReadTheDocs) +- docs8 (style linter) + +These requirements are already present in setup.cfg and `./configure --docs` installs them. + +Now you can build the HTML documents locally:: + + source venv/bin/activate + cd docs + make html + +Assuming that your Sphinx installation was successful, Sphinx should build a local instance of the +documentation .html files:: + + open build/html/index.html + +.. note:: + + In case this command did not work, for example on Ubuntu 18.04 you may get a message like “Couldn’t + get a file descriptor referring to the console”, try: + + :: + + see build/html/index.html + +You now have a local build of the AboutCode documents. + +.. _contrib_doc_share_improvements: + +Share Document Improvements +--------------------------- + +Ensure that you have the latest files:: + + git pull + git status + +Before commiting changes run Continious Integration Scripts locally to run tests. Refer +:ref:`doc_ci` for instructions on the same. + +Follow standard git procedures to upload your new and modified files. The following commands are +examples:: + + git status + git add source/index.rst + git add source/how-to-scan.rst + git status + git commit -m "New how-to document that explains how to scan" + git status + git push + git status + +The Scancode-Toolkit webhook with ReadTheDocs should rebuild the documentation after your +Pull Request is Merged. + +Refer the `Pro Git Book `_ available online for Git tutorials +covering more complex topics on Branching, Merging, Rebasing etc. + +.. _doc_ci: + +Continuous Integration +---------------------- + +The documentations are checked on every new commit through Travis-CI, so that common errors are +avoided and documentation standards are enforced. Travis-CI presently checks for these 3 aspects +of the documentation : + +1. Successful Builds (By using ``sphinx-build``) +2. No Broken Links (By Using ``link-check``) +3. Linting Errors (By Using ``Doc8``) + +So run these scripts at your local system before creating a Pull Request:: + + cd docs + ./scripts/sphinx_build_link_check.sh + ./scripts/doc8_style_check.sh + +If you don't have permission to run the scripts, run:: + + chmod u+x ./scripts/doc8_style_check.sh + +.. _doc_style_docs8: + +Style Checks Using ``Doc8`` +--------------------------- + +How To Run Style Tests +^^^^^^^^^^^^^^^^^^^^^^ + +In the project root, run the following commands:: + + $ cd docs + $ ./scripts/doc8_style_check.sh + +A sample output is:: + + Scanning... + Validating... + docs/source/misc/licence_policy_plugin.rst:37: D002 Trailing whitespace + docs/source/misc/faq.rst:45: D003 Tabulation used for indentation + docs/source/misc/faq.rst:9: D001 Line too long + docs/source/misc/support.rst:6: D005 No newline at end of file + ======== + Total files scanned = 34 + Total files ignored = 0 + Total accumulated errors = 326 + Detailed error counts: + - CheckCarriageReturn = 0 + - CheckIndentationNoTab = 75 + - CheckMaxLineLength = 190 + - CheckNewlineEndOfFile = 13 + - CheckTrailingWhitespace = 47 + - CheckValidity = 1 + +Now fix the errors and run again till there isn't any style error in the documentation. + +What is Checked? +^^^^^^^^^^^^^^^^ + +PyCQA is an Organization for code quality tools (and plugins) for the Python programming language. +Doc8 is a sub-project of the same Organization. Refer this `README `_ for more details. + +What is checked: + + - invalid rst format - D000 + - lines should not be longer than 100 characters - D001 + + - RST exception: line with no whitespace except in the beginning + - RST exception: lines with http or https URLs + - RST exception: literal blocks + - RST exception: rst target directives + + - no trailing whitespace - D002 + - no tabulation for indentation - D003 + - no carriage returns (use UNIX newlines) - D004 + - no newline at end of file - D005 + +.. _doc_interspinx: + +Interspinx +---------- + +ScanCode toolkit documentation uses `Intersphinx `_ +to link to other Sphinx Documentations, to maintain links to other Aboutcode Projects. + +To link sections in the same documentation, standart reST labels are used. Refer +`Cross-Referencing `_ for more information. + +For example:: + + .. _my-reference-label: + + Section to cross-reference + -------------------------- + + This is the text of the section. + + It refers to the section itself, see :ref:`my-reference-label`. + +Now, using Intersphinx, you can create these labels in one Sphinx Documentation and then referance +these labels from another Sphinx Documentation, hosted in different locations. + +You just have to add the following in the ``conf.py`` file for your Sphinx Documentation, where you +want to add the links:: + + extensions = [ + 'sphinx.ext.intersphinx' + ] + + intersphinx_mapping = {'aboutcode': ('https://aboutcode.readthedocs.io/en/latest/', None)} + +To show all Intersphinx links and their targets of an Intersphinx mapping file, run:: + + python -msphinx.ext.intersphinx https://aboutcode.readthedocs.io/en/latest/objects.inv + +.. WARNING:: + + ``python -msphinx.ext.intersphinx https://aboutcode.readthedocs.io/objects.inv`` will give + error. + +This enables you to create links to the ``aboutcode`` Documentation in your own Documentation, +where you modified the configuration file. Links can be added like this:: + + For more details refer :ref:`aboutcode:doc_style_guide`. + +You can also not use the ``aboutcode`` label assigned to all links from aboutcode.readthedocs.io, +if you don't have a label having the same name in your Sphinx Documentation. Example:: + + For more details refer :ref:`doc_style_guide`. + +If you have a label in your documentation which is also present in the documentation linked by +Intersphinx, and you link to that label, it will create a link to the local label. + +For more information, refer this tutorial named +`Using Intersphinx `_. + +.. _doc_style_conv: + +Style Conventions for the Documentaion +-------------------------------------- + +1. Headings + + (`Refer `_) + Normally, there are no heading levels assigned to certain characters as the structure is + determined from the succession of headings. However, this convention is used in Python’s Style + Guide for documenting which you may follow: + + # with overline, for parts + + * with overline, for chapters + + =, for sections + + -, for subsections + + ^, for sub-subsections + + ", for paragraphs + +2. Heading Underlines + + Do not use underlines that are longer/shorter than the title headline itself. As in: + + :: + + Correct : + + Extra Style Checks + ------------------ + + Incorrect : + + Extra Style Checks + ------------------------ + +.. note:: + + Underlines shorter than the Title text generates Errors on sphinx-build. + + +3. Internal Links + + Using ``:ref:`` is advised over standard reStructuredText links to sections (like + ```Section title`_``) because it works across files, when section headings are changed, will + raise warnings if incorrect, and works for all builders that support cross-references. + However, external links are created by using the standard ```Section title`_`` method. + +4. Eliminate Redundancy + + If a section/file has to be repeated somewhere else, do not write the exact same section/file + twice. Use ``.. include: ../README.rst`` instead. Here, ``../`` refers to the documentation + root, so file location can be used accordingly. This enables us to link documents from other + upstream folders. + +5. Using ``:ref:`` only when necessary + + Use ``:ref:`` to create internal links only when needed, i.e. it is referenced somewhere. + Do not create references for all the sections and then only reference some of them, because + this created unnecessary references. This also generates ERROR in ``restructuredtext-lint``. + +6. Spelling + + You should check for spelling errors before you push changes. `Aspell `_ + is a GNU project Command Line tool you can use for this purpose. Download and install Aspell, + then execute ``aspell check `` for all the files changed. Be careful about not + changing commands or other stuff as Aspell gives prompts for a lot of them. Also delete the + temporary ``.bak`` files generated. Refer the `manual `_ for more + information on how to use. + +7. Notes and Warning Snippets + + Every ``Note`` and ``Warning`` sections are to be kept in ``rst_snippets/note_snippets/`` and + ``rst_snippets/warning_snippets/`` and then included to eliminate redundancy, as these are + frequently used in multiple files. + +Converting from Markdown +------------------------ + +If you want to convert a ``.md`` file to a ``.rst`` file, this `tool `_ +does it pretty well. You'd still have to clean up and check for errors as this contains a lot of +bugs. But this is definitely better than converting everything by yourself. + +This will be helpful in converting GitHub wiki's (Markdown Files) to reStructuredtext files for +Sphinx/ReadTheDocs hosting. diff --git a/docs/source/index.rst b/docs/source/index.rst index 67fcf21..eb63717 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -5,7 +5,8 @@ Welcome to nexb-skeleton's documentation! :maxdepth: 2 :caption: Contents: - skeleton/index + skeleton-usage + contribute/contrib_doc Indices and tables ================== diff --git a/docs/skeleton-usage.rst b/docs/source/skeleton-usage.rst similarity index 98% rename from docs/skeleton-usage.rst rename to docs/source/skeleton-usage.rst index ad9b9ff..cde23dc 100644 --- a/docs/skeleton-usage.rst +++ b/docs/source/skeleton-usage.rst @@ -73,11 +73,13 @@ To generate requirements.txt: python etc/scripts/gen_requirements.py -s venv/lib/python/site-packages/ -Replace \ with the version number of the Python being used, for example: ``venv/lib/python3.6/site-packages/`` +Replace \ with the version number of the Python being used, for example: +``venv/lib/python3.6/site-packages/`` To generate requirements-dev.txt after requirements.txt has been generated: .. code-block:: bash + ./configure --dev python etc/scripts/gen_requirements_dev.py -s venv/lib/python/site-packages/ diff --git a/docs/source/skeleton/index.rst b/docs/source/skeleton/index.rst deleted file mode 100644 index f99cdec..0000000 --- a/docs/source/skeleton/index.rst +++ /dev/null @@ -1,15 +0,0 @@ -# Docs Structure Guide -# Rst docs - https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html -# -# 1. Place docs in folders under source for different sections -# 2. Link them by adding individual index files in each section -# to the main index, and then files for each section to their -# respective index files. -# 3. Use `.. include` statements to include other .rst files -# or part of them, or use hyperlinks to a section of the docs, -# to get rid of repetition. -# https://docutils.sourceforge.io/docs/ref/rst/directives.html#including-an-external-document-fragment -# -# Note: Replace these guide/placeholder docs - -.. include:: ../../../README.rst From 5431ee548c5bbfaf289f93281611d61c777aa575 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Thu, 28 Apr 2022 12:43:20 -0700 Subject: [PATCH 30/41] Properly check for existance of thirdparty dir Signed-off-by: Jono Yang --- configure | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure b/configure index 715c99f..d1b4fda 100755 --- a/configure +++ b/configure @@ -55,7 +55,7 @@ CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin ################################ # Thirdparty package locations and index handling # Find packages from the local thirdparty directory or from thirdparty.aboutcode.org -if [ -f "$CFG_ROOT_DIR/thirdparty" ]; then +if [ -d "$CFG_ROOT_DIR/thirdparty" ]; then PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty" fi PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS --find-links https://thirdparty.aboutcode.org/pypi/simple/links.html" From c44c4424d7fa82723c2aac7f2a79f380411e1949 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 29 Apr 2022 15:41:07 +0200 Subject: [PATCH 31/41] Improve GH action documentation Signed-off-by: Philippe Ombredanne --- .github/workflows/pypi-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 188497e..b0a8d97 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -1,4 +1,4 @@ -name: Release library as a PyPI wheel and sdist on tag +name: Release library as a PyPI wheel and sdist on GH release creation on: release: From 99ba101572144cc5e5d42f2136985eb91163a46a Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 29 Apr 2022 15:50:02 +0200 Subject: [PATCH 32/41] Use Python 3.9 as a base for actions Signed-off-by: Philippe Ombredanne --- .github/workflows/docs-ci.yml | 2 +- .github/workflows/pypi-release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml index 656d624..18a44aa 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs-ci.yml @@ -9,7 +9,7 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: [3.7] + python-version: [3.9] steps: - name: Checkout code diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index b0a8d97..3a4fe27 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -13,7 +13,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: 3.9 - name: Install pypa/build run: python -m pip install build --user - name: Build a binary wheel and a source tarball From 00f4fe76dad5f0fa8efc6768af99079389c583ac Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Fri, 29 Apr 2022 14:31:34 -0700 Subject: [PATCH 33/41] Remove variable from string in fetch_thirdparty.py * The variable `environment` is not used when fetching sdists Signed-off-by: Jono Yang --- etc/scripts/fetch_thirdparty.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py index 22147b2..042266c 100644 --- a/etc/scripts/fetch_thirdparty.py +++ b/etc/scripts/fetch_thirdparty.py @@ -269,11 +269,11 @@ def fetch_thirdparty( if TRACE: if not fetched: print( - f" ====> Sdist already available: {name}=={version} on: {environment}" + f" ====> Sdist already available: {name}=={version}" ) else: print( - f" ====> Sdist fetched: {fetched} for {name}=={version} on: {environment}" + f" ====> Sdist fetched: {fetched} for {name}=={version}" ) except utils_thirdparty.DistributionNotFound as e: From 5d48c1cbb7262455cc2c51958833ddb9ecb2bbce Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 2 May 2022 11:33:50 +0200 Subject: [PATCH 34/41] Improve thirdparty scripts Ensure that site-package dir exists. Other minor adjustments from a scancode-toolkit release Signed-off-by: Philippe Ombredanne --- etc/scripts/fetch_thirdparty.py | 7 ++-- etc/scripts/utils_requirements.py | 3 ++ etc/scripts/utils_thirdparty.py | 66 +++++++++++++++++++------------ 3 files changed, 47 insertions(+), 29 deletions(-) diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py index 042266c..f31e81f 100644 --- a/etc/scripts/fetch_thirdparty.py +++ b/etc/scripts/fetch_thirdparty.py @@ -18,7 +18,8 @@ import utils_thirdparty import utils_requirements -TRACE = True +TRACE = False +TRACE_DEEP = False @click.command() @@ -204,7 +205,7 @@ def fetch_thirdparty( existing_wheels = None if existing_wheels: - if TRACE: + if TRACE_DEEP: print( f"====> Wheels already available: {name}=={version} on: {environment}: {existing_package.wheels!r}" ) @@ -213,7 +214,7 @@ def fetch_thirdparty( else: continue - if TRACE: + if TRACE_DEEP: print(f"Fetching wheel for: {name}=={version} on: {environment}") try: diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index fbc456d..069b465 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -8,6 +8,7 @@ # See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # +import os import re import subprocess @@ -110,6 +111,8 @@ def get_installed_reqs(site_packages_dir): Return the installed pip requirements as text found in `site_packages_dir` as a text. """ + if not os.path.exists(site_packages_dir): + raise Exception(f"site_packages directort: {site_packages_dir!r} does not exists") # Also include these packages in the output with --all: wheel, distribute, # setuptools, pip args = ["pip", "freeze", "--exclude-editable", "--all", "--path", site_packages_dir] diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 829cf8c..4c40969 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -8,8 +8,8 @@ # See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # -from collections import defaultdict import email +import functools import itertools import os import re @@ -18,6 +18,8 @@ import tempfile import time import urllib +from collections import defaultdict +from urllib.parse import quote_plus import attr import license_expression @@ -29,7 +31,6 @@ from commoncode.text import python_safe_name from packaging import tags as packaging_tags from packaging import version as packaging_version -from urllib.parse import quote_plus import utils_pip_compatibility_tags from utils_requirements import load_requirements @@ -111,7 +112,7 @@ """ -TRACE = True +TRACE = False TRACE_DEEP = False TRACE_ULTRA_DEEP = False @@ -233,7 +234,7 @@ def download_wheel( tuple of lists of (fetched_wheel_filenames, existing_wheel_filenames) """ if TRACE_DEEP: - print(f" download_wheel: {name}=={version}: {environment}") + print(f" download_wheel: {name}=={version}: {environment} and index_urls: {index_urls}") fetched_wheel_filenames = [] existing_wheel_filenames = [] @@ -311,6 +312,7 @@ def download_sdist( raise DistributionNotFound(f"Failed to fetch sdist: {name}=={version}: No sources found") +@functools.cache def get_package_versions( name, version=None, @@ -321,15 +323,28 @@ def get_package_versions( repository ``index_urls`` list of URLs. If ``version`` is not provided, return the latest available versions. """ + found = [] + not_found = [] for index_url in index_urls: try: repo = get_pypi_repo(index_url) package = repo.get_package(name, version) + if package: - yield package + found.append((package, index_url)) + else: + not_found.append((name, version, index_url)) except RemoteNotFetchedException as e: - print(f"Failed to fetch PyPI package {name} @ {version} info from {index_url}: {e}") + if TRACE_ULTRA_DEEP: + print(f"Failed to fetch PyPI package {name} @ {version} info from {index_url}: {e}") + not_found.append((name, version, index_url)) + + if not found: + raise Exception(f"No PyPI package {name} @ {version} found!") + for package, index_url in found: + print(f"Fetched PyPI package {package.name} @ {package.version} info from {index_url}") + yield package ################################################################################ # @@ -546,14 +561,14 @@ def get_best_download_url( If none is found, return a synthetic remote URL. """ for index_url in index_urls: - pypi_package = get_pypi_package( + pypi_package = get_pypi_package_data( name=self.normalized_name, version=self.version, index_url=index_url, ) if pypi_package: if isinstance(pypi_package, tuple): - raise Exception("############", repr(pypi_package)) + raise Exception("############", repr(pypi_package), self.normalized_name, self.version, index_url) try: pypi_url = pypi_package.get_url_for_filename(self.filename) except Exception as e: @@ -1450,7 +1465,7 @@ def get_name_version(cls, name, version, packages): nvs = [p for p in cls.get_versions(name, packages) if p.version == version] if not nvs: - return name, version + return if len(nvs) == 1: return nvs[0] @@ -1494,8 +1509,8 @@ def dists_from_paths_or_urls(cls, paths_or_urls): >>> assert expected == result """ dists = [] - if TRACE_DEEP: - print(" ###paths_or_urls:", paths_or_urls) + if TRACE_ULTRA_DEEP: + print(" ###paths_or_urls:", paths_or_urls) installable = [f for f in paths_or_urls if f.endswith(EXTENSIONS_INSTALLABLE)] for path_or_url in installable: try: @@ -1618,7 +1633,6 @@ def tags(self): ) ) - ################################################################################ # # PyPI repo and link index for package wheels and sources @@ -1657,7 +1671,10 @@ def get_versions(self, name): The list may be empty. """ name = name and NameVer.normalize_name(name) - self._populate_links_and_packages(name) + try: + self._populate_links_and_packages(name) + except Exception as e: + print(f" ==> Cannot find versions of {name}: {e}") return self.packages_by_normalized_name.get(name, []) def get_latest_version(self, name): @@ -1703,7 +1720,7 @@ def _fetch_links(self, name, _LINKS={}): _LINKS[index_url] = [l for l in links if l.endswith(EXTENSIONS)] links = _LINKS[index_url] - if TRACE_DEEP: + if TRACE_ULTRA_DEEP: print(f" Found links {links!r}") return links @@ -1803,7 +1820,6 @@ def from_url(cls, url=ABOUT_BASE_URL, _LINKS_REPO={}): _LINKS_REPO[url] = cls(url=url) return _LINKS_REPO[url] - ################################################################################ # Globals for remote repos to be lazily created and cached on first use for the # life of the session together with some convenience functions. @@ -1824,19 +1840,21 @@ def get_pypi_repo(index_url, _PYPI_REPO={}): return _PYPI_REPO[index_url] -def get_pypi_package(name, version, index_url, verbose=TRACE_DEEP): +@functools.cache +def get_pypi_package_data(name, version, index_url, verbose=TRACE_DEEP): """ Return a PypiPackage or None. """ try: + if verbose: + print(f" get_pypi_package_data: Fetching {name} @ {version} info from {index_url}") package = get_pypi_repo(index_url).get_package(name, version) if verbose: - print(f" get_pypi_package: {name} @ {version} info from {index_url}: {package}") + print(f" get_pypi_package_data: Fetched: {package}") return package except RemoteNotFetchedException as e: - print(f"Failed to fetch PyPI package {name} @ {version} info from {index_url}: {e}") - + print(f" get_pypi_package_data: Failed to fetch PyPI package {name} @ {version} info from {index_url}: {e}") ################################################################################ # @@ -1998,7 +2016,6 @@ def fetch_and_save_path_or_url( fo.write(content) return content - ################################################################################ # Requirements processing ################################################################################ @@ -2036,7 +2053,6 @@ def get_required_packages( print(" get_required_packages: name:", name, "version:", version) yield repo.get_package(name, version) - ################################################################################ # Functions to update or fetch ABOUT and license files ################################################################################ @@ -2115,7 +2131,6 @@ def get_other_dists(_package, _dist): for p in packages_by_name[local_package.name] if p.version != local_package.version ] - other_local_version = other_local_packages and other_local_packages[-1] if other_local_version: latest_local_dists = list(other_local_version.get_distributions()) @@ -2187,7 +2202,6 @@ def get_other_dists(_package, _dist): lic_errs = "\n".join(lic_errs) print(f"Failed to fetch some licenses:: {lic_errs}") - ################################################################################ # # Functions to build new Python wheels including native on multiple OSes @@ -2318,9 +2332,9 @@ def build_wheels_locally_if_pure_python( "--wheel-dir", wheel_dir, ] - + deps - + verbose - + [requirements_specifier] + +deps + +verbose + +[requirements_specifier] ) print(f"Building local wheels for: {requirements_specifier}") From 6a3c5b0b9e351b9c8730836c2db878a1540cbe2a Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 7 May 2022 19:44:52 +0200 Subject: [PATCH 35/41] Update thirdparty fetching utilities These were buggy in some corner cases. They have been updated such that: * --latest-version works. * we can reliable fetch combinations of wheels and sdists for multiple OS combos at once * we now support macOS universal wheels (for ARM CPUs) Caching is now simpler: we have essentially a single file-based cache under .cache. PyPI indexes are fetched and not cached, unless the new --use-cached-index is used which can be useful when fetching many thirdparty in a short timeframe. The first PyPI repository in a list has precendence and we never fetch from other repositories if we find wheels and sdsists there. This avoid pounding too much on the self-hosted repo. Signed-off-by: Philippe Ombredanne --- etc/scripts/check_thirdparty.py | 6 +- etc/scripts/fetch_thirdparty.py | 188 +++---- etc/scripts/gen_pypi_simple.py | 22 +- etc/scripts/requirements.txt | 5 +- etc/scripts/utils_requirements.py | 19 +- etc/scripts/utils_thirdparty.py | 899 +++++++++++++----------------- 6 files changed, 480 insertions(+), 659 deletions(-) diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py index 0f04b34..b052f25 100644 --- a/etc/scripts/check_thirdparty.py +++ b/etc/scripts/check_thirdparty.py @@ -16,7 +16,7 @@ @click.command() @click.option( "-d", - "--dest_dir", + "--dest", type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), required=True, help="Path to the thirdparty directory to check.", @@ -35,7 +35,7 @@ ) @click.help_option("-h", "--help") def check_thirdparty_dir( - dest_dir, + dest, wheels, sdists, ): @@ -45,7 +45,7 @@ def check_thirdparty_dir( # check for problems print(f"==> CHECK FOR PROBLEMS") utils_thirdparty.find_problems( - dest_dir=dest_dir, + dest_dir=dest, report_missing_sources=sdists, report_missing_wheels=wheels, ) diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py index f31e81f..26d520f 100644 --- a/etc/scripts/fetch_thirdparty.py +++ b/etc/scripts/fetch_thirdparty.py @@ -100,11 +100,17 @@ "index_urls", type=str, metavar="INDEX", - default=utils_thirdparty.PYPI_INDEXES, + default=utils_thirdparty.PYPI_INDEX_URLS, show_default=True, multiple=True, help="PyPI index URL(s) to use for wheels and sources, in order of preferences.", ) +@click.option( + "--use-cached-index", + is_flag=True, + help="Use on disk cached PyPI indexes list of packages and versions and do not refetch if present.", +) + @click.help_option("-h", "--help") def fetch_thirdparty( requirements_files, @@ -116,9 +122,10 @@ def fetch_thirdparty( wheels, sdists, index_urls, + use_cached_index, ): """ - Download to --dest-dir THIRDPARTY_DIR the PyPI wheels, source distributions, + Download to --dest THIRDPARTY_DIR the PyPI wheels, source distributions, and their ABOUT metadata, license and notices files. Download the PyPI packages listed in the combination of: @@ -126,16 +133,23 @@ def fetch_thirdparty( - the pip name==version --specifier SPECIFIER(s) - any pre-existing wheels or sdsists found in --dest-dir THIRDPARTY_DIR. - Download wheels with the --wheels option for the ``--python-version`` PYVER(s) - and ``--operating_system`` OS(s) combinations defaulting to all supported combinations. + Download wheels with the --wheels option for the ``--python-version`` + PYVER(s) and ``--operating_system`` OS(s) combinations defaulting to all + supported combinations. Download sdists tarballs with the --sdists option. - Generate or Download .ABOUT, .LICENSE and .NOTICE files for all the wheels and sources fetched. + Generate or Download .ABOUT, .LICENSE and .NOTICE files for all the wheels + and sources fetched. - Download wheels and sdists the provided PyPI simple --index-url INDEX(s) URLs. + Download from the provided PyPI simple --index-url INDEX(s) URLs. """ + if not (wheels or sdists): + print("Error: one or both of --wheels and --sdists is required.") + sys.exit(1) + print(f"COLLECTING REQUIRED NAMES & VERSIONS FROM {dest_dir}") + existing_packages_by_nv = { (package.name, package.version): package for package in utils_thirdparty.get_local_packages(directory=dest_dir) @@ -151,134 +165,88 @@ def fetch_thirdparty( required_name_versions.update(nvs) for specifier in specifiers: - nv = utils_requirements.get_name_version( + nv = utils_requirements.get_required_name_version( requirement=specifier, with_unpinned=latest_version, ) required_name_versions.add(nv) + if latest_version: + names = set(name for name, _version in sorted(required_name_versions)) + required_name_versions = {(n, None) for n in names} + if not required_name_versions: print("Error: no requirements requested.") sys.exit(1) - if not os.listdir(dest_dir) and not (wheels or sdists): - print("Error: one or both of --wheels and --sdists is required.") - sys.exit(1) - - if latest_version: - latest_name_versions = set() - names = set(name for name, _version in sorted(required_name_versions)) - for name in sorted(names): - latests = utils_thirdparty.PypiPackage.sorted( - utils_thirdparty.get_package_versions( - name=name, version=None, index_urls=index_urls - ) - ) - if not latests: - print(f"No distribution found for: {name}") - continue - latest = latests[-1] - latest_name_versions.add((latest.name, latest.version)) - required_name_versions = latest_name_versions - - if TRACE: - print("required_name_versions:", required_name_versions) + if TRACE_DEEP: + print("required_name_versions:") + for n, v in required_name_versions: + print(f" {n} @ {v}") + # create the environments matrix we need for wheels + environments = None if wheels: - # create the environments matrix we need for wheels evts = itertools.product(python_versions, operating_systems) environments = [utils_thirdparty.Environment.from_pyver_and_os(pyv, os) for pyv, os in evts] - wheels_not_found = {} - sdists_not_found = {} - # iterate over requirements, one at a time + # Collect PyPI repos + repos = [] + for index_url in index_urls: + index_url = index_url.strip("/") + existing = utils_thirdparty.DEFAULT_PYPI_REPOS_BY_URL.get(index_url) + if existing: + existing.use_cached_index = use_cached_index + repos.append(existing) + else: + repo = utils_thirdparty.PypiSimpleRepository( + index_url=index_url, + use_cached_index=use_cached_index, + ) + repos.append(repo) + + wheels_fetched = [] + wheels_not_found = [] + + sdists_fetched = [] + sdists_not_found = [] + for name, version in sorted(required_name_versions): nv = name, version - existing_package = existing_packages_by_nv.get(nv) + print(f"Processing: {name} @ {version}") if wheels: for environment in environments: - if existing_package: - existing_wheels = list( - existing_package.get_supported_wheels(environment=environment) - ) - else: - existing_wheels = None - - if existing_wheels: - if TRACE_DEEP: - print( - f"====> Wheels already available: {name}=={version} on: {environment}: {existing_package.wheels!r}" - ) - if all(w.is_pure() for w in existing_wheels): - break - else: - continue - - if TRACE_DEEP: - print(f"Fetching wheel for: {name}=={version} on: {environment}") - - try: - ( - fetched_wheel_filenames, - existing_wheel_filenames, - ) = utils_thirdparty.download_wheel( - name=name, - version=version, - environment=environment, - dest_dir=dest_dir, - index_urls=index_urls, - ) - if TRACE: - if existing_wheel_filenames: - print( - f" ====> Wheels already available: {name}=={version} on: {environment}" - ) - for whl in existing_wheel_filenames: - print(f" {whl}") - if fetched_wheel_filenames: - print(f" ====> Wheels fetched: {name}=={version} on: {environment}") - for whl in fetched_wheel_filenames: - print(f" {whl}") - - fwfns = fetched_wheel_filenames + existing_wheel_filenames - - if all(utils_thirdparty.Wheel.from_filename(f).is_pure() for f in fwfns): - break - - except utils_thirdparty.DistributionNotFound as e: - wheels_not_found[f"{name}=={version}"] = str(e) - - if sdists: - if existing_package and existing_package.sdist: if TRACE: - print( - f" ====> Sdist already available: {name}=={version}: {existing_package.sdist!r}" - ) - continue - - if TRACE: - print(f" Fetching sdist for: {name}=={version}") - - try: - fetched = utils_thirdparty.download_sdist( + print(f" ==> Fetching wheel for envt: {environment}") + fwfns = utils_thirdparty.download_wheel( name=name, version=version, + environment=environment, dest_dir=dest_dir, - index_urls=index_urls, + repos=repos, ) + if fwfns: + wheels_fetched.extend(fwfns) + else: + wheels_not_found.append(f"{name}=={version} for: {environment}") + if TRACE: + print(f" NOT FOUND") + if sdists: + if TRACE: + print(f" ==> Fetching sdist: {name}=={version}") + fetched = utils_thirdparty.download_sdist( + name=name, + version=version, + dest_dir=dest_dir, + repos=repos, + ) + if fetched: + sdists_fetched.append(fetched) + else: + sdists_not_found.append(f"{name}=={version}") if TRACE: - if not fetched: - print( - f" ====> Sdist already available: {name}=={version}" - ) - else: - print( - f" ====> Sdist fetched: {fetched} for {name}=={version}" - ) - - except utils_thirdparty.DistributionNotFound as e: - sdists_not_found[f"{name}=={version}"] = str(e) + print(f" NOT FOUND") if wheels and wheels_not_found: print(f"==> MISSING WHEELS") @@ -291,7 +259,7 @@ def fetch_thirdparty( print(f" {sd}") print(f"==> FETCHING OR CREATING ABOUT AND LICENSE FILES") - utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir) + utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir, use_cached_index=use_cached_index) utils_thirdparty.clean_about_files(dest_dir=dest_dir) # check for problems diff --git a/etc/scripts/gen_pypi_simple.py b/etc/scripts/gen_pypi_simple.py index 8de2b96..03312ab 100644 --- a/etc/scripts/gen_pypi_simple.py +++ b/etc/scripts/gen_pypi_simple.py @@ -25,26 +25,26 @@ class InvalidDistributionFilename(Exception): def get_package_name_from_filename(filename): """ - Return the package name extracted from a package ``filename``. - Optionally ``normalize`` the name according to distribution name rules. + Return the normalized package name extracted from a package ``filename``. + Normalization is done according to distribution name rules. Raise an ``InvalidDistributionFilename`` if the ``filename`` is invalid:: >>> get_package_name_from_filename("foo-1.2.3_rc1.tar.gz") 'foo' - >>> get_package_name_from_filename("foo-bar-1.2-py27-none-any.whl") + >>> get_package_name_from_filename("foo_bar-1.2-py27-none-any.whl") 'foo-bar' >>> get_package_name_from_filename("Cython-0.17.2-cp26-none-linux_x86_64.whl") 'cython' >>> get_package_name_from_filename("python_ldap-2.4.19-cp27-none-macosx_10_10_x86_64.whl") 'python-ldap' - >>> get_package_name_from_filename("foo.whl") - Traceback (most recent call last): - ... - InvalidDistributionFilename: ... - >>> get_package_name_from_filename("foo.png") - Traceback (most recent call last): - ... - InvalidFilePackageName: ... + >>> try: + ... get_package_name_from_filename("foo.whl") + ... except InvalidDistributionFilename: + ... pass + >>> try: + ... get_package_name_from_filename("foo.png") + ... except InvalidDistributionFilename: + ... pass """ if not filename or not filename.endswith(dist_exts): raise InvalidDistributionFilename(filename) diff --git a/etc/scripts/requirements.txt b/etc/scripts/requirements.txt index 6591e49..ebb404b 100644 --- a/etc/scripts/requirements.txt +++ b/etc/scripts/requirements.txt @@ -1,12 +1,11 @@ aboutcode_toolkit -github-release-retry2 attrs commoncode click requests saneyaml -romp pip setuptools twine -wheel \ No newline at end of file +wheel +build \ No newline at end of file diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index 069b465..7c99a33 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -8,7 +8,6 @@ # See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # -import os import re import subprocess @@ -42,23 +41,23 @@ def get_required_name_versions(requirement_lines, with_unpinned=False): if req_line.startswith("-") or (not with_unpinned and not "==" in req_line): print(f"Requirement line is not supported: ignored: {req_line}") continue - yield get_name_version(requirement=req_line, with_unpinned=with_unpinned) + yield get_required_name_version(requirement=req_line, with_unpinned=with_unpinned) -def get_name_version(requirement, with_unpinned=False): +def get_required_name_version(requirement, with_unpinned=False): """ Return a (name, version) tuple given a`requirement` specifier string. Requirement version must be pinned. If ``with_unpinned`` is True, unpinned requirements are accepted and only the name portion is returned. For example: - >>> assert get_name_version("foo==1.2.3") == ("foo", "1.2.3") - >>> assert get_name_version("fooA==1.2.3.DEV1") == ("fooa", "1.2.3.dev1") - >>> assert get_name_version("foo==1.2.3", with_unpinned=False) == ("foo", "1.2.3") - >>> assert get_name_version("foo", with_unpinned=True) == ("foo", "") - >>> assert get_name_version("foo>=1.2", with_unpinned=True) == ("foo", ""), get_name_version("foo>=1.2") + >>> assert get_required_name_version("foo==1.2.3") == ("foo", "1.2.3") + >>> assert get_required_name_version("fooA==1.2.3.DEV1") == ("fooa", "1.2.3.dev1") + >>> assert get_required_name_version("foo==1.2.3", with_unpinned=False) == ("foo", "1.2.3") + >>> assert get_required_name_version("foo", with_unpinned=True) == ("foo", "") + >>> assert get_required_name_version("foo>=1.2", with_unpinned=True) == ("foo", ""), get_required_name_version("foo>=1.2") >>> try: - ... assert not get_name_version("foo", with_unpinned=False) + ... assert not get_required_name_version("foo", with_unpinned=False) ... except Exception as e: ... assert "Requirement version must be pinned" in str(e) """ @@ -112,7 +111,7 @@ def get_installed_reqs(site_packages_dir): as a text. """ if not os.path.exists(site_packages_dir): - raise Exception(f"site_packages directort: {site_packages_dir!r} does not exists") + raise Exception(f"site_packages directory: {site_packages_dir!r} does not exists") # Also include these packages in the output with --all: wheel, distribute, # setuptools, pip args = ["pip", "freeze", "--exclude-editable", "--all", "--path", site_packages_dir] diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 4c40969..9cbda37 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -9,7 +9,6 @@ # See https://aboutcode.org for more information about nexB OSS projects. # import email -import functools import itertools import os import re @@ -33,7 +32,6 @@ from packaging import version as packaging_version import utils_pip_compatibility_tags -from utils_requirements import load_requirements """ Utilities to manage Python thirparty libraries source, binaries and metadata in @@ -169,6 +167,16 @@ def get_python_dot_version(version): "macosx_10_15_x86_64", "macosx_11_0_x86_64", "macosx_11_intel", + "macosx_11_0_x86_64", + "macosx_11_intel", + "macosx_10_9_universal2", + "macosx_10_10_universal2", + "macosx_10_11_universal2", + "macosx_10_12_universal2", + "macosx_10_13_universal2", + "macosx_10_14_universal2", + "macosx_10_15_universal2", + "macosx_11_0_universal2", # 'macosx_11_0_arm64', ], "windows": [ @@ -179,18 +187,19 @@ def get_python_dot_version(version): THIRDPARTY_DIR = "thirdparty" CACHE_THIRDPARTY_DIR = ".cache/thirdparty" -ABOUT_BASE_URL = "https://thirdparty.aboutcode.org/pypi" +################################################################################ +ABOUT_BASE_URL = "https://thirdparty.aboutcode.org/pypi" ABOUT_PYPI_SIMPLE_URL = f"{ABOUT_BASE_URL}/simple" ABOUT_LINKS_URL = f"{ABOUT_PYPI_SIMPLE_URL}/links.html" - PYPI_SIMPLE_URL = "https://pypi.org/simple" -PYPI_INDEXES = (PYPI_SIMPLE_URL, ABOUT_PYPI_SIMPLE_URL) +PYPI_INDEX_URLS = (PYPI_SIMPLE_URL, ABOUT_PYPI_SIMPLE_URL) + +################################################################################ EXTENSIONS_APP = (".pyz",) EXTENSIONS_SDIST = ( ".tar.gz", - ".tar.bz2", ".zip", ".tar.xz", ) @@ -217,134 +226,90 @@ class DistributionNotFound(Exception): pass -def download_wheel( - name, - version, - environment, - dest_dir=THIRDPARTY_DIR, - index_urls=PYPI_INDEXES, -): +def download_wheel(name, version, environment, dest_dir=THIRDPARTY_DIR, repos=tuple()): """ Download the wheels binary distribution(s) of package ``name`` and - ``version`` matching the ``environment`` Environment constraints from the - PyPI simple repository ``index_urls`` list of URLs into the ``dest_dir`` - directory. + ``version`` matching the ``environment`` Environment constraints into the + ``dest_dir`` directory. Return a list of fetched_wheel_filenames, possibly + empty. - Raise a DistributionNotFound if no wheel is not found. Otherwise, return a - tuple of lists of (fetched_wheel_filenames, existing_wheel_filenames) + Use the first PyPI simple repository from a list of ``repos`` that contains this wheel. """ if TRACE_DEEP: - print(f" download_wheel: {name}=={version}: {environment} and index_urls: {index_urls}") + print(f" download_wheel: {name}=={version} for envt: {environment}") - fetched_wheel_filenames = [] - existing_wheel_filenames = [] - try: - for pypi_package in get_package_versions( - name=name, - version=version, - index_urls=index_urls, - ): - if not pypi_package.wheels: - continue - - supported_wheels = list(pypi_package.get_supported_wheels(environment=environment)) - if not supported_wheels: - continue + if not repos: + repos = DEFAULT_PYPI_REPOS - for wheel in supported_wheels: - if os.path.exists(os.path.join(dest_dir, wheel.filename)): - # do not refetch - existing_wheel_filenames.append(wheel.filename) - continue + fetched_wheel_filenames = [] - if TRACE: - print(f" Fetching wheel from index: {wheel.download_url}") - fetched_wheel_filename = wheel.download(dest_dir=dest_dir) - fetched_wheel_filenames.add(fetched_wheel_filename) + for repo in repos: + package = repo.get_package_version(name=name, version=version) + if not package: + if TRACE_DEEP: + print(f" download_wheel: No package in {repo.index_url} for {name}=={version}") + continue + supported_wheels = list(package.get_supported_wheels(environment=environment)) + if not supported_wheels: + if TRACE_DEEP: + print( + f" download_wheel: No supported wheel for {name}=={version}: {environment} " + ) + continue - except Exception as e: - raise DistributionNotFound(f"Failed to fetch wheel: {name}=={version}: {e}") from e + for wheel in supported_wheels: + if TRACE_DEEP: + print( + f" download_wheel: Getting wheel from index (or cache): {wheel.download_url}" + ) + fetched_wheel_filename = wheel.download(dest_dir=dest_dir) + fetched_wheel_filenames.append(fetched_wheel_filename) - if not fetched_wheel_filenames and not existing_wheel_filenames: - raise DistributionNotFound(f"Failed to fetch wheel: {name}=={version}: No wheel found") + if fetched_wheel_filenames: + # do not futher fetch from other repos if we find in first, typically PyPI + break - return fetched_wheel_filenames, existing_wheel_filenames + return fetched_wheel_filenames -def download_sdist( - name, - version, - dest_dir=THIRDPARTY_DIR, - index_urls=PYPI_INDEXES, -): +def download_sdist(name, version, dest_dir=THIRDPARTY_DIR, repos=tuple()): """ Download the sdist source distribution of package ``name`` and ``version`` - from the PyPI simple repository ``index_urls`` list of URLs into the - ``dest_dir`` directory. + into the ``dest_dir`` directory. Return a fetched filename or None. - Raise a DistributionNotFound if this was not found. Return the filename if - downloaded and False if not downloaded because it already exists. + Use the first PyPI simple repository from a list of ``repos`` that contains + this sdist. """ - if TRACE_DEEP: - print(f"download_sdist: {name}=={version}: ") - - try: - for pypi_package in get_package_versions( - name=name, - version=version, - index_urls=index_urls, - ): - if not pypi_package.sdist: - continue - - if os.path.exists(os.path.join(dest_dir, pypi_package.sdist.filename)): - # do not refetch - return False - if TRACE: - print(f" Fetching sources from index: {pypi_package.sdist.download_url}") - fetched = pypi_package.sdist.download(dest_dir=dest_dir) - if fetched: - return pypi_package.sdist.filename + if TRACE: + print(f" download_sdist: {name}=={version}") - except Exception as e: - raise DistributionNotFound(f"Failed to fetch sdist: {name}=={version}: {e}") from e + if not repos: + repos = DEFAULT_PYPI_REPOS - raise DistributionNotFound(f"Failed to fetch sdist: {name}=={version}: No sources found") + fetched_sdist_filename = None + for repo in repos: + package = repo.get_package_version(name=name, version=version) -@functools.cache -def get_package_versions( - name, - version=None, - index_urls=PYPI_INDEXES, -): - """ - Yield PypiPackages with ``name`` and ``version`` from the PyPI simple - repository ``index_urls`` list of URLs. - If ``version`` is not provided, return the latest available versions. - """ - found = [] - not_found = [] - for index_url in index_urls: - try: - repo = get_pypi_repo(index_url) - package = repo.get_package(name, version) + if not package: + if TRACE_DEEP: + print(f" download_sdist: No package in {repo.index_url} for {name}=={version}") + continue + sdist = package.sdist + if not sdist: + if TRACE_DEEP: + print(f" download_sdist: No sdist for {name}=={version}") + continue - if package: - found.append((package, index_url)) - else: - not_found.append((name, version, index_url)) - except RemoteNotFetchedException as e: - if TRACE_ULTRA_DEEP: - print(f"Failed to fetch PyPI package {name} @ {version} info from {index_url}: {e}") - not_found.append((name, version, index_url)) + if TRACE_DEEP: + print(f" download_sdist: Getting sdist from index (or cache): {sdist.download_url}") + fetched_sdist_filename = package.sdist.download(dest_dir=dest_dir) - if not found: - raise Exception(f"No PyPI package {name} @ {version} found!") + if fetched_sdist_filename: + # do not futher fetch from other repos if we find in first, typically PyPI + break - for package, index_url in found: - print(f"Fetched PyPI package {package.name} @ {package.version} info from {index_url}") - yield package + return fetched_sdist_filename ################################################################################ # @@ -377,17 +342,6 @@ def normalize_name(name): """ return name and re.sub(r"[-_.]+", "-", name).lower() or name - @staticmethod - def standardize_name(name): - """ - Return a standardized package name, e.g. lowercased and using - not _ - """ - return name and re.sub(r"[-_]+", "-", name).lower() or name - - @property - def name_ver(self): - return f"{self.name}-{self.version}" - def sortable_name_version(self): """ Return a tuple of values to sort by name, then version. @@ -403,7 +357,7 @@ def sorted(cls, namevers): @attr.attributes class Distribution(NameVer): - # field names that can be updated from another dist of mapping + # field names that can be updated from another Distribution or mapping updatable_fields = [ "license_expression", "copyright", @@ -421,6 +375,13 @@ class Distribution(NameVer): metadata=dict(help="File name."), ) + path_or_url = attr.ib( + repr=False, + type=str, + default="", + metadata=dict(help="Path or URL"), + ) + sha256 = attr.ib( repr=False, type=str, @@ -545,36 +506,50 @@ def package_url(self): """ Return a Package URL string of self. """ - return str(packageurl.PackageURL(**self.purl_identifiers())) + return str( + packageurl.PackageURL( + type=self.type, + namespace=self.namespace, + name=self.name, + version=self.version, + subpath=self.subpath, + qualifiers=self.qualifiers, + ) + ) @property def download_url(self): return self.get_best_download_url() - def get_best_download_url( - self, - index_urls=tuple([PYPI_SIMPLE_URL, ABOUT_PYPI_SIMPLE_URL]), - ): + def get_best_download_url(self, repos=tuple()): """ - Return the best download URL for this distribution where best means that - PyPI is better and our selfhosted repo URLs are second. - If none is found, return a synthetic remote URL. + Return the best download URL for this distribution where best means this + is the first URL found for this distribution found in the list of + ``repos``. + + If none is found, return a synthetic PyPI remote URL. """ - for index_url in index_urls: - pypi_package = get_pypi_package_data( - name=self.normalized_name, - version=self.version, - index_url=index_url, - ) - if pypi_package: - if isinstance(pypi_package, tuple): - raise Exception("############", repr(pypi_package), self.normalized_name, self.version, index_url) - try: - pypi_url = pypi_package.get_url_for_filename(self.filename) - except Exception as e: - raise Exception(repr(pypi_package)) from e - if pypi_url: - return pypi_url + + if not repos: + repos = DEFAULT_PYPI_REPOS + + for repo in repos: + package = repo.get_package_version(name=self.name, version=self.version) + if not package: + if TRACE: + print( + f" get_best_download_url: {self.name}=={self.version} " + f"not found in {repo.index_url}" + ) + continue + pypi_url = package.get_url_for_filename(self.filename) + if pypi_url: + return pypi_url + else: + if TRACE: + print( + f" get_best_download_url: {self.filename} not found in {repo.index_url}" + ) def download(self, dest_dir=THIRDPARTY_DIR): """ @@ -582,16 +557,17 @@ def download(self, dest_dir=THIRDPARTY_DIR): Return the fetched filename. """ assert self.filename - if TRACE: + if TRACE_DEEP: print( f"Fetching distribution of {self.name}=={self.version}:", self.filename, ) - fetch_and_save_path_or_url( - filename=self.filename, - dest_dir=dest_dir, + # FIXME: + fetch_and_save( path_or_url=self.path_or_url, + dest_dir=dest_dir, + filename=self.filename, as_text=False, ) return self.filename @@ -616,7 +592,7 @@ def notice_download_url(self): def from_path_or_url(cls, path_or_url): """ Return a distribution built from the data found in the filename of a - `path_or_url` string. Raise an exception if this is not a valid + ``path_or_url`` string. Raise an exception if this is not a valid filename. """ filename = os.path.basename(path_or_url.strip("/")) @@ -647,47 +623,6 @@ def from_filename(cls, filename): clazz = cls.get_dist_class(filename) return clazz.from_filename(filename) - def purl_identifiers(self, skinny=False): - """ - Return a mapping of non-empty identifier name/values for the purl - fields. If skinny is True, only inlucde type, namespace and name. - """ - identifiers = dict( - type=self.type, - namespace=self.namespace, - name=self.name, - ) - - if not skinny: - identifiers.update( - version=self.version, - subpath=self.subpath, - qualifiers=self.qualifiers, - ) - - return {k: v for k, v in sorted(identifiers.items()) if v} - - def identifiers(self, purl_as_fields=True): - """ - Return a mapping of non-empty identifier name/values. - Return each purl fields separately if purl_as_fields is True. - Otherwise return a package_url string for the purl. - """ - if purl_as_fields: - identifiers = self.purl_identifiers() - else: - identifiers = dict(package_url=self.package_url) - - identifiers.update( - download_url=self.download_url, - filename=self.filename, - md5=self.md5, - sha1=self.sha1, - package_url=self.package_url, - ) - - return {k: v for k, v in sorted(identifiers.items()) if v} - def has_key_metadata(self): """ Return True if this distribution has key metadata required for basic attribution. @@ -817,7 +752,7 @@ def load_remote_about_data(self): NOTICE file if any. Return True if the data was updated. """ try: - about_text = fetch_content_from_path_or_url_through_cache( + about_text = CACHE.get( path_or_url=self.about_download_url, as_text=True, ) @@ -831,7 +766,7 @@ def load_remote_about_data(self): notice_file = about_data.pop("notice_file", None) if notice_file: try: - notice_text = fetch_content_from_path_or_url_through_cache( + notice_text = CACHE.get( path_or_url=self.notice_download_url, as_text=True, ) @@ -882,12 +817,12 @@ def get_license_keys(self): return ["unknown"] return keys - def fetch_license_files(self, dest_dir=THIRDPARTY_DIR): + def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): """ Fetch license files if missing in `dest_dir`. Return True if license files were fetched. """ - urls = LinksRepository.from_url().links + urls = LinksRepository.from_url(use_cached_index=use_cached_index).links errors = [] extra_lic_names = [l.get("file") for l in self.extra_data.get("licenses", {})] extra_lic_names += [self.extra_data.get("license_file")] @@ -902,10 +837,10 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR): # try remotely first lic_url = get_license_link_for_filename(filename=filename, urls=urls) - fetch_and_save_path_or_url( - filename=filename, - dest_dir=dest_dir, + fetch_and_save( path_or_url=lic_url, + dest_dir=dest_dir, + filename=filename, as_text=True, ) if TRACE: @@ -915,10 +850,10 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR): try: # try licensedb second lic_url = f"{LICENSEDB_API_URL}/{filename}" - fetch_and_save_path_or_url( - filename=filename, - dest_dir=dest_dir, + fetch_and_save( path_or_url=lic_url, + dest_dir=dest_dir, + filename=filename, as_text=True, ) if TRACE: @@ -1077,6 +1012,84 @@ class InvalidDistributionFilename(Exception): pass +def get_sdist_name_ver_ext(filename): + """ + Return a (name, version, extension) if filename is a valid sdist name. Some legacy + binary builds have weird names. Return False otherwise. + + In particular they do not use PEP440 compliant versions and/or mix tags, os + and arch names in tarball names and versions: + + >>> assert get_sdist_name_ver_ext("intbitset-1.3.tar.gz") + >>> assert not get_sdist_name_ver_ext("intbitset-1.3.linux-x86_64.tar.gz") + >>> assert get_sdist_name_ver_ext("intbitset-1.4a.tar.gz") + >>> assert get_sdist_name_ver_ext("intbitset-1.4a.zip") + >>> assert not get_sdist_name_ver_ext("intbitset-2.0.linux-x86_64.tar.gz") + >>> assert get_sdist_name_ver_ext("intbitset-2.0.tar.gz") + >>> assert not get_sdist_name_ver_ext("intbitset-2.1-1.src.rpm") + >>> assert not get_sdist_name_ver_ext("intbitset-2.1-1.x86_64.rpm") + >>> assert not get_sdist_name_ver_ext("intbitset-2.1.linux-x86_64.tar.gz") + >>> assert not get_sdist_name_ver_ext("cffi-1.2.0-1.tar.gz") + >>> assert not get_sdist_name_ver_ext("html5lib-1.0-reupload.tar.gz") + >>> assert not get_sdist_name_ver_ext("selenium-2.0-dev-9429.tar.gz") + >>> assert not get_sdist_name_ver_ext("testfixtures-1.8.0dev-r4464.tar.gz") + """ + name_ver = None + extension = None + + for ext in EXTENSIONS_SDIST: + if filename.endswith(ext): + name_ver, extension, _ = filename.rpartition(ext) + break + + if not extension or not name_ver: + return False + + name, _, version = name_ver.rpartition("-") + + if not name or not version: + return False + + # weird version + if any( + w in version + for w in ( + "x86_64", + "i386", + ) + ): + return False + + # all char versions + if version.isalpha(): + return False + + # non-pep 440 version + if "-" in version: + return False + + # single version + if version.isdigit() and len(version) == 1: + return False + + # r1 version + if len(version) == 2 and version[0]=="r" and version[1].isdigit(): + return False + + # dotless version (but calver is OK) + if "." not in version and len(version) < 3: + return False + + # version with dashes selenium-2.0-dev-9429.tar.gz + if name.endswith(("dev",)) and "." not in version: + return False + # version pre or post, old legacy + if version.startswith(("beta", "rc", "pre", "post", "final")): + return False + + return name, version, extension + + @attr.attributes class Sdist(Distribution): @@ -1093,21 +1106,11 @@ def from_filename(cls, filename): Return a Sdist object built from a filename. Raise an exception if this is not a valid sdist filename """ - name_ver = None - extension = None - - for ext in EXTENSIONS_SDIST: - if filename.endswith(ext): - name_ver, extension, _ = filename.rpartition(ext) - break - - if not extension or not name_ver: + name_ver_ext = get_sdist_name_ver_ext(filename) + if not name_ver_ext: raise InvalidDistributionFilename(filename) - name, _, version = name_ver.rpartition("-") - - if not name or not version: - raise InvalidDistributionFilename(filename) + name, version, extension = name_ver_ext return cls( type="pypi", @@ -1295,8 +1298,8 @@ def is_pure_wheel(filename): @attr.attributes class PypiPackage(NameVer): """ - A Python package with its "distributions", e.g. wheels and source - distribution , ABOUT files and licenses or notices. + A Python package contains one or more wheels and one source distribution + from a repository. """ sdist = attr.ib( @@ -1313,16 +1316,6 @@ class PypiPackage(NameVer): metadata=dict(help="List of Wheel for this package"), ) - @property - def specifier(self): - """ - A requirement specifier for this package - """ - if self.version: - return f"{self.name}=={self.version}" - else: - return self.name - def get_supported_wheels(self, environment, verbose=TRACE_ULTRA_DEEP): """ Yield all the Wheel of this package supported and compatible with the @@ -1404,17 +1397,20 @@ def packages_from_dir(cls, directory): Yield PypiPackages built from files found in at directory path. """ base = os.path.abspath(directory) + paths = [os.path.join(base, f) for f in os.listdir(base) if f.endswith(EXTENSIONS)] + if TRACE_ULTRA_DEEP: print("packages_from_dir: paths:", paths) - return cls.packages_from_many_paths_or_urls(paths) + return PypiPackage.packages_from_many_paths_or_urls(paths) @classmethod def packages_from_many_paths_or_urls(cls, paths_or_urls): """ Yield PypiPackages built from a list of paths or URLs. + These are sorted by name and then by version from oldest to newest. """ - dists = cls.dists_from_paths_or_urls(paths_or_urls) + dists = PypiPackage.dists_from_paths_or_urls(paths_or_urls) if TRACE_ULTRA_DEEP: print("packages_from_many_paths_or_urls: dists:", dists) @@ -1429,54 +1425,11 @@ def packages_from_many_paths_or_urls(cls, paths_or_urls): print("packages_from_many_paths_or_urls", package) yield package - @classmethod - def get_versions(cls, name, packages): - """ - Return a subset list of package versions from a list of `packages` that - match PypiPackage `name`. - The list is sorted by version from oldest to most recent. - """ - norm_name = NameVer.normalize_name(name) - versions = [p for p in packages if p.normalized_name == norm_name] - return cls.sorted(versions) - - @classmethod - def get_latest_version(cls, name, packages): - """ - Return the latest version of PypiPackage `name` from a list of `packages`. - """ - versions = cls.get_versions(name, packages) - if not versions: - return - return versions[-1] - - @classmethod - def get_name_version(cls, name, version, packages): - """ - Return the PypiPackage with `name` and `version` from a list of `packages` - or None if it is not found. - If `version` is None, return the latest version found. - """ - if TRACE_ULTRA_DEEP: - print("get_name_version:", name, version, packages) - if not version: - return cls.get_latest_version(name, packages) - - nvs = [p for p in cls.get_versions(name, packages) if p.version == version] - - if not nvs: - return - - if len(nvs) == 1: - return nvs[0] - - raise Exception(f"More than one PypiPackage with {name}=={version}") - @classmethod def dists_from_paths_or_urls(cls, paths_or_urls): """ Return a list of Distribution given a list of - `paths_or_urls` to wheels or source distributions. + ``paths_or_urls`` to wheels or source distributions. Each Distribution receives two extra attributes: - the path_or_url it was created from @@ -1488,25 +1441,20 @@ def dists_from_paths_or_urls(cls, paths_or_urls): ... bitarray-0.8.1-cp36-cp36m-macosx_10_9_x86_64.macosx_10_10_x86_64.whl ... bitarray-0.8.1-cp36-cp36m-win_amd64.whl ... https://example.com/bar/bitarray-0.8.1.tar.gz - ... bitarray-0.8.1.tar.gz.ABOUT bit.LICENSE'''.split() - >>> result = list(PypiPackage.dists_from_paths_or_urls(paths_or_urls)) + ... bitarray-0.8.1.tar.gz.ABOUT + ... bit.LICENSE'''.split() + >>> results = list(PypiPackage.dists_from_paths_or_urls(paths_or_urls)) >>> for r in results: - ... r.filename = '' - ... r.path_or_url = '' - >>> expected = [ - ... Wheel(name='bitarray', version='0.8.1', build='', - ... python_versions=['cp36'], abis=['cp36m'], - ... platforms=['linux_x86_64']), - ... Wheel(name='bitarray', version='0.8.1', build='', - ... python_versions=['cp36'], abis=['cp36m'], - ... platforms=['macosx_10_9_x86_64', 'macosx_10_10_x86_64']), - ... Wheel(name='bitarray', version='0.8.1', build='', - ... python_versions=['cp36'], abis=['cp36m'], - ... platforms=['win_amd64']), - ... Sdist(name='bitarray', version='0.8.1'), - ... Sdist(name='bitarray', version='0.8.1') - ... ] - >>> assert expected == result + ... print(r.__class__.__name__, r.name, r.version) + ... if isinstance(r, Wheel): + ... print(" ", ", ".join(r.python_versions), ", ".join(r.platforms)) + Wheel bitarray 0.8.1 + cp36 linux_x86_64 + Wheel bitarray 0.8.1 + cp36 macosx_10_9_x86_64, macosx_10_10_x86_64 + Wheel bitarray 0.8.1 + cp36 win_amd64 + Sdist bitarray 0.8.1 """ dists = [] if TRACE_ULTRA_DEEP: @@ -1518,7 +1466,14 @@ def dists_from_paths_or_urls(cls, paths_or_urls): dists.append(dist) if TRACE_DEEP: print( - " ===> dists_from_paths_or_urls:", dist, "with URL:", dist.download_url + " ===> dists_from_paths_or_urls:", + dist, + "\n ", + "with URL:", + dist.download_url, + "\n ", + "from URL:", + path_or_url, ) except InvalidDistributionFilename: if TRACE_DEEP: @@ -1653,101 +1608,105 @@ class PypiSimpleRepository: metadata=dict(help="Base PyPI simple URL for this index."), ) - packages_by_normalized_name = attr.ib( + # we keep a nested mapping of PypiPackage that has this shape: + # {name: {version: PypiPackage, version: PypiPackage, etc} + # the inner versions mapping is sorted by version from oldest to newest + + packages = attr.ib( type=dict, - default=attr.Factory(lambda: defaultdict(list)), - metadata=dict(help="Mapping of {package name: [package objects]} available in this repo"), + default=attr.Factory(lambda: defaultdict(dict)), + metadata=dict( + help="Mapping of {name: {version: PypiPackage, version: PypiPackage, etc} available in this repo" + ), ) - packages_by_normalized_name_version = attr.ib( - type=dict, - default=attr.Factory(dict), - metadata=dict(help="Mapping of {(name, version): package object} available in this repo"), + fetched_package_normalized_names = attr.ib( + type=set, + default=attr.Factory(set), + metadata=dict(help="A set of already fetched package normalized names."), ) - def get_versions(self, name): + use_cached_index = attr.ib( + type=bool, + default=False, + metadata=dict(help="If True, use any existing on-disk cached PyPI index files. Otherwise, fetch and cache."), + ) + + def _get_package_versions_map(self, name): """ - Return a list of all available PypiPackage version for this package name. - The list may be empty. + Return a mapping of all available PypiPackage version for this package name. + The mapping may be empty. It is ordered by version from oldest to newest """ - name = name and NameVer.normalize_name(name) - try: - self._populate_links_and_packages(name) - except Exception as e: - print(f" ==> Cannot find versions of {name}: {e}") - return self.packages_by_normalized_name.get(name, []) + assert name + normalized_name = NameVer.normalize_name(name) + versions = self.packages[normalized_name] + if not versions and normalized_name not in self.fetched_package_normalized_names: + self.fetched_package_normalized_names.add(normalized_name) + try: + links = self.fetch_links(normalized_name=normalized_name) + # note that thsi is sorted so the mapping is also sorted + versions = { + package.version: package + for package in PypiPackage.packages_from_many_paths_or_urls(paths_or_urls=links) + } + self.packages[normalized_name] = versions + except RemoteNotFetchedException as e: + if TRACE: + print(f"failed to fetch package name: {name} from: {self.index_url}:\n{e}") + + if not versions and TRACE: + print(f"WARNING: package {name} not found in repo: {self.index_url}") + + return versions - def get_latest_version(self, name): + def get_package_versions(self, name): """ - Return the latest PypiPackage version for this package name or None. + Return a mapping of all available PypiPackage version as{version: + package} for this package name. The mapping may be empty but not None. + It is sorted by version from oldest to newest. """ - versions = self.get_versions(name) - return PypiPackage.get_latest_version(name, versions) + return dict(self._get_package_versions_map(name)) - def get_package(self, name, version): + def get_package_version(self, name, version=None): """ Return the PypiPackage with name and version or None. + Return the latest PypiPackage version if version is None. """ - versions = self.get_versions(name) - if TRACE_DEEP: - print("PypiPackage.get_package:versions:", versions) - return PypiPackage.get_name_version(name, version, versions) + if not version: + versions = list(self._get_package_versions_map(name).values()) + return versions and versions[-1] + else: + return self._get_package_versions_map(name).get(version) - def _fetch_links(self, name, _LINKS={}): + def fetch_links(self, normalized_name): """ Return a list of download link URLs found in a PyPI simple index for package name using the `index_url` of this repository. """ - name = name and NameVer.normalize_name(name) - index_url = self.index_url - - name = name and NameVer.normalize_name(name) - index_url = index_url.strip("/") - index_url = f"{index_url}/{name}" - - if TRACE_DEEP: - print( - f" Finding links for {name!r} from PyPI index: {index_url} : cached?:", - index_url in _LINKS, - ) - - if index_url not in _LINKS: - text = fetch_content_from_path_or_url_through_cache(path_or_url=index_url, as_text=True) - links = collect_urls(text) - # TODO: keep sha256 - links = [l.partition("#sha256=") for l in links] - links = [url for url, _, _sha256 in links] - _LINKS[index_url] = [l for l in links if l.endswith(EXTENSIONS)] - - links = _LINKS[index_url] - if TRACE_ULTRA_DEEP: - print(f" Found links {links!r}") + package_url = f"{self.index_url}/{normalized_name}" + text = CACHE.get( + path_or_url=package_url, + as_text=True, + force=not self.use_cached_index, + ) + links = collect_urls(text) + # TODO: keep sha256 + links = [l.partition("#sha256=") for l in links] + links = [url for url, _, _sha256 in links] return links - def _populate_links_and_packages(self, name): - name = name and NameVer.normalize_name(name) - - if TRACE_DEEP: - print("PypiPackage._populate_links_and_packages:name:", name) - - links = self._fetch_links(name) - packages = list(PypiPackage.packages_from_many_paths_or_urls(paths_or_urls=links)) - - if TRACE_DEEP: - print("PypiPackage._populate_links_and_packages:packages:", packages) - self.packages_by_normalized_name[name] = packages - - for p in packages: - name = name and NameVer.normalize_name(p.name) - self.packages_by_normalized_name_version[(name, p.version)] = p +PYPI_PUBLIC_REPO = PypiSimpleRepository(index_url=PYPI_SIMPLE_URL) +PYPI_SELFHOSTED_REPO = PypiSimpleRepository(index_url=ABOUT_PYPI_SIMPLE_URL) +DEFAULT_PYPI_REPOS = PYPI_PUBLIC_REPO, PYPI_SELFHOSTED_REPO +DEFAULT_PYPI_REPOS_BY_URL = {r.index_url: r for r in DEFAULT_PYPI_REPOS} @attr.attributes class LinksRepository: """ - Represents a simple links repository such an HTTP directory listing or a - page with links. + Represents a simple links repository such an HTTP directory listing or an + HTML page with links. """ url = attr.ib( @@ -1762,14 +1721,23 @@ class LinksRepository: metadata=dict(help="List of links available in this repo"), ) + use_cached_index = attr.ib( + type=bool, + default=False, + metadata=dict(help="If True, use any existing on-disk cached index files. Otherwise, fetch and cache."), + ) + def __attrs_post_init__(self): if not self.links: self.links = self.find_links() - def find_links(self): + def find_links(self, _CACHE=[]): """ Return a list of link URLs found in the HTML page at `self.url` """ + if _CACHE: + return _CACHE + links_url = self.url if TRACE_DEEP: print(f"Finding links from: {links_url}") @@ -1781,9 +1749,10 @@ def find_links(self): if TRACE_DEEP: print(f"Base URL {base_url}") - text = fetch_content_from_path_or_url_through_cache( + text = CACHE.get( path_or_url=links_url, as_text=True, + force=not self.use_cached_index, ) links = [] @@ -1812,12 +1781,13 @@ def find_links(self): if TRACE: print(f"Found {len(links)} links at {links_url}") + _CACHE.extend(links) return links @classmethod - def from_url(cls, url=ABOUT_BASE_URL, _LINKS_REPO={}): + def from_url(cls, url=ABOUT_BASE_URL, _LINKS_REPO={}, use_cached_index=False): if url not in _LINKS_REPO: - _LINKS_REPO[url] = cls(url=url) + _LINKS_REPO[url] = cls(url=url, use_cached_index=use_cached_index) return _LINKS_REPO[url] ################################################################################ @@ -1833,29 +1803,6 @@ def get_local_packages(directory=THIRDPARTY_DIR): """ return list(PypiPackage.packages_from_dir(directory=directory)) - -def get_pypi_repo(index_url, _PYPI_REPO={}): - if index_url not in _PYPI_REPO: - _PYPI_REPO[index_url] = PypiSimpleRepository(index_url=index_url) - return _PYPI_REPO[index_url] - - -@functools.cache -def get_pypi_package_data(name, version, index_url, verbose=TRACE_DEEP): - """ - Return a PypiPackage or None. - """ - try: - if verbose: - print(f" get_pypi_package_data: Fetching {name} @ {version} info from {index_url}") - package = get_pypi_repo(index_url).get_package(name, version) - if verbose: - print(f" get_pypi_package_data: Fetched: {package}") - return package - - except RemoteNotFetchedException as e: - print(f" get_pypi_package_data: Failed to fetch PyPI package {name} @ {version} info from {index_url}: {e}") - ################################################################################ # # Basic file and URL-based operations using a persistent file-based Cache @@ -1875,34 +1822,40 @@ class Cache: def __attrs_post_init__(self): os.makedirs(self.directory, exist_ok=True) - def clear(self): - shutil.rmtree(self.directory) - - def get(self, path_or_url, as_text=True): + def get(self, path_or_url, as_text=True, force=False): """ - Get a file from a `path_or_url` through the cache. - `path_or_url` can be a path or a URL to a file. + Return the content fetched from a ``path_or_url`` through the cache. + Raise an Exception on errors. Treats the content as text if as_text is + True otherwise as treat as binary. `path_or_url` can be a path or a URL + to a file. """ cache_key = quote_plus(path_or_url.strip("/")) cached = os.path.join(self.directory, cache_key) - if not os.path.exists(cached): + if force or not os.path.exists(cached): + if TRACE_DEEP: + print(f" FILE CACHE MISS: {path_or_url}") content = get_file_content(path_or_url=path_or_url, as_text=as_text) wmode = "w" if as_text else "wb" with open(cached, wmode) as fo: fo.write(content) return content else: + if TRACE_DEEP: + print(f" FILE CACHE HIT: {path_or_url}") return get_local_file_content(path=cached, as_text=as_text) +CACHE = Cache() + + def get_file_content(path_or_url, as_text=True): """ Fetch and return the content at `path_or_url` from either a local path or a remote URL. Return the content as bytes is `as_text` is False. """ if path_or_url.startswith("https://"): - if TRACE: + if TRACE_DEEP: print(f"Fetching: {path_or_url}") _headers, content = get_remote_file_content(url=path_or_url, as_text=as_text) return content @@ -1954,7 +1907,7 @@ def get_remote_file_content( # using a GET with stream=True ensure we get the the final header from # several redirects and that we can ignore content there. A HEAD request may # not get us this last header - print(f" DOWNLOADING {url}") + print(f" DOWNLOADING: {url}") with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: status = response.status_code if status != requests.codes.ok: # NOQA @@ -1978,35 +1931,19 @@ def get_remote_file_content( return response.headers, response.text if as_text else response.content -def fetch_content_from_path_or_url_through_cache( +def fetch_and_save( path_or_url, - as_text=True, - cache=Cache(), -): - """ - Return the content from fetching at path or URL. Raise an Exception on - errors. Treats the content as text if as_text is True otherwise as treat as - binary. Use the provided file cache. This is the main entry for using the - cache. - - Note: the `cache` argument is a global, though it does not really matter - since it does not hold any state which is only kept on disk. - """ - return cache.get(path_or_url=path_or_url, as_text=as_text) - - -def fetch_and_save_path_or_url( - filename, dest_dir, - path_or_url, + filename, as_text=True, ): """ - Return the content from fetching the `filename` file name at URL or path - and save to `dest_dir`. Raise an Exception on errors. Treats the content as - text if as_text is True otherwise as treat as binary. + Fetch content at ``path_or_url`` URL or path and save this to + ``dest_dir/filername``. Return the fetched content. Raise an Exception on + errors. Treats the content as text if as_text is True otherwise as treat as + binary. """ - content = fetch_content_from_path_or_url_through_cache( + content = CACHE.get( path_or_url=path_or_url, as_text=as_text, ) @@ -2017,44 +1954,9 @@ def fetch_and_save_path_or_url( return content ################################################################################ -# Requirements processing -################################################################################ - - -def get_required_remote_packages( - requirements_file="requirements.txt", - index_url=PYPI_SIMPLE_URL, -): - """ - Yield tuple of (name, version, PypiPackage) for packages listed in the - `requirements_file` requirements file and found in the PyPI index - ``index_url`` URL. - """ - required_name_versions = load_requirements(requirements_file=requirements_file) - return get_required_packages(required_name_versions=required_name_versions, index_url=index_url) - - -def get_required_packages( - required_name_versions, - index_url=PYPI_SIMPLE_URL, -): - """ - Yield tuple of (name, version) or a PypiPackage for package name/version - listed in the ``required_name_versions`` list and found in the PyPI index - ``index_url`` URL. - """ - if TRACE: - print("get_required_packages", index_url) - - repo = get_pypi_repo(index_url=index_url) - - for name, version in required_name_versions: - if TRACE: - print(" get_required_packages: name:", name, "version:", version) - yield repo.get_package(name, version) - -################################################################################ +# # Functions to update or fetch ABOUT and license files +# ################################################################################ @@ -2075,7 +1977,7 @@ def clean_about_files( local_dist.save_about_and_notice_files(dest_dir) -def fetch_abouts_and_licenses(dest_dir=THIRDPARTY_DIR): +def fetch_abouts_and_licenses(dest_dir=THIRDPARTY_DIR, use_cached_index=False): """ Given a thirdparty dir, add missing ABOUT. LICENSE and NOTICE files using best efforts: @@ -2085,6 +1987,8 @@ def fetch_abouts_and_licenses(dest_dir=THIRDPARTY_DIR): - derive from existing distribution with same name and latest version that would have such ABOUT file - extract ABOUT file data from distributions PKGINFO or METADATA files + + Use available existing on-disk cached index if use_cached_index is True. """ def get_other_dists(_package, _dist): @@ -2094,7 +1998,6 @@ def get_other_dists(_package, _dist): """ return [d for d in _package.get_distributions() if d != _dist] - selfhosted_repo = get_pypi_repo(index_url=ABOUT_PYPI_SIMPLE_URL) local_packages = get_local_packages(directory=dest_dir) packages_by_name = defaultdict(list) for local_package in local_packages: @@ -2110,7 +2013,7 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files(dest_dir=dest_dir) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue # lets try to get from another dist of the same local package @@ -2122,7 +2025,7 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files(dest_dir=dest_dir) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue # try to get another version of the same package that is not our version @@ -2148,7 +2051,7 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files(dest_dir=dest_dir) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue # lets try to fetch remotely @@ -2157,14 +2060,16 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files(dest_dir=dest_dir) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue # try to get a latest version of the same package that is not our version + # and that is in our self hosted repo + lpv = local_package.version + lpn = local_package.name + other_remote_packages = [ - p - for p in selfhosted_repo.get_versions(local_package.name) - if p.version != local_package.version + p for v, p in PYPI_SELFHOSTED_REPO.get_package_versions(lpn).items() if v != lpv ] latest_version = other_remote_packages and other_remote_packages[-1] @@ -2184,7 +2089,7 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files(dest_dir=dest_dir) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue # try to get data from pkginfo (no license though) @@ -2194,7 +2099,7 @@ def get_other_dists(_package, _dist): # if local_dist.has_key_metadata() or not local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir) - lic_errs = local_dist.fetch_license_files(dest_dir) + lic_errs = local_dist.fetch_license_files(dest_dir, use_cached_index=use_cached_index) if not local_dist.has_key_metadata(): print(f"Unable to add essential ABOUT data for: {local_dist}") @@ -2305,66 +2210,16 @@ def download_wheels_with_pip( downloaded = existing ^ set(os.listdir(dest_dir)) return sorted(downloaded), error - -def build_wheels_locally_if_pure_python( - requirements_specifier, - with_deps=False, - verbose=False, - dest_dir=THIRDPARTY_DIR, -): - """ - Given pip `requirements_specifier` string (such as package names or as - name==version), build the corresponding binary wheel(s) locally. - - If all these are "pure" Python wheels that run on all Python 3 versions and - operating systems, copy them back in `dest_dir` if they do not exists there - - Return a tuple of (True if all wheels are "pure", list of built wheel file names) - """ - deps = [] if with_deps else ["--no-deps"] - verbose = ["--verbose"] if verbose else [] - - wheel_dir = tempfile.mkdtemp(prefix="scancode-release-wheels-local-") - cli_args = ( - [ - "pip", - "wheel", - "--wheel-dir", - wheel_dir, - ] - +deps - +verbose - +[requirements_specifier] - ) - - print(f"Building local wheels for: {requirements_specifier}") - print(f"Using command:", " ".join(cli_args)) - call(cli_args) - - built = os.listdir(wheel_dir) - if not built: - return [] - - all_pure = all(is_pure_wheel(bwfn) for bwfn in built) - - if not all_pure: - print(f" Some wheels are not pure") - - print(f" Copying local wheels") - pure_built = [] - for bwfn in built: - owfn = os.path.join(dest_dir, bwfn) - if not os.path.exists(owfn): - nwfn = os.path.join(wheel_dir, bwfn) - fileutils.copyfile(nwfn, owfn) - pure_built.append(bwfn) - print(f" Built local wheel: {bwfn}") - return all_pure, pure_built +################################################################################ +# +# Functions to check for problems +# +################################################################################ def check_about(dest_dir=THIRDPARTY_DIR): try: - subprocess.check_output(f"about check {dest_dir}".split()) + subprocess.check_output(f"venv/bin/about check {dest_dir}".split()) except subprocess.CalledProcessError as cpe: print() print("Invalid ABOUT files:") From ff348f5fa2882091ee892c3a0760edba3a63bd53 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 9 May 2022 22:58:46 +0200 Subject: [PATCH 36/41] Format code with black Signed-off-by: Philippe Ombredanne --- docs/source/conf.py | 12 +++++------- etc/scripts/fetch_thirdparty.py | 1 - etc/scripts/utils_thirdparty.py | 31 +++++++++++++++++++++++-------- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 62bca04..d5435e7 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -28,7 +28,7 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ -'sphinx.ext.intersphinx', + "sphinx.ext.intersphinx", ] # This points to aboutcode.readthedocs.io @@ -36,8 +36,8 @@ # Link was created at commit - https://github.com/nexB/aboutcode/commit/faea9fcf3248f8f198844fe34d43833224ac4a83 intersphinx_mapping = { - 'aboutcode': ('https://aboutcode.readthedocs.io/en/latest/', None), - 'scancode-workbench': ('https://scancode-workbench.readthedocs.io/en/develop/', None), + "aboutcode": ("https://aboutcode.readthedocs.io/en/latest/", None), + "scancode-workbench": ("https://scancode-workbench.readthedocs.io/en/develop/", None), } @@ -62,7 +62,7 @@ # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] -master_doc = 'index' +master_doc = "index" html_context = { "display_github": True, @@ -72,9 +72,7 @@ "conf_py_path": "/docs/source/", # path in the checkout to the docs root } -html_css_files = [ - '_static/theme_overrides.css' - ] +html_css_files = ["_static/theme_overrides.css"] # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py index 26d520f..89d17de 100644 --- a/etc/scripts/fetch_thirdparty.py +++ b/etc/scripts/fetch_thirdparty.py @@ -110,7 +110,6 @@ is_flag=True, help="Use on disk cached PyPI indexes list of packages and versions and do not refetch if present.", ) - @click.help_option("-h", "--help") def fetch_thirdparty( requirements_files, diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 9cbda37..2d6f3e4 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -311,6 +311,7 @@ def download_sdist(name, version, dest_dir=THIRDPARTY_DIR, repos=tuple()): return fetched_sdist_filename + ################################################################################ # # Core models @@ -1064,16 +1065,16 @@ def get_sdist_name_ver_ext(filename): if version.isalpha(): return False - # non-pep 440 version + # non-pep 440 version if "-" in version: return False - # single version + # single version if version.isdigit() and len(version) == 1: return False - # r1 version - if len(version) == 2 and version[0]=="r" and version[1].isdigit(): + # r1 version + if len(version) == 2 and version[0] == "r" and version[1].isdigit(): return False # dotless version (but calver is OK) @@ -1588,6 +1589,7 @@ def tags(self): ) ) + ################################################################################ # # PyPI repo and link index for package wheels and sources @@ -1629,7 +1631,9 @@ class PypiSimpleRepository: use_cached_index = attr.ib( type=bool, default=False, - metadata=dict(help="If True, use any existing on-disk cached PyPI index files. Otherwise, fetch and cache."), + metadata=dict( + help="If True, use any existing on-disk cached PyPI index files. Otherwise, fetch and cache." + ), ) def _get_package_versions_map(self, name): @@ -1724,7 +1728,9 @@ class LinksRepository: use_cached_index = attr.ib( type=bool, default=False, - metadata=dict(help="If True, use any existing on-disk cached index files. Otherwise, fetch and cache."), + metadata=dict( + help="If True, use any existing on-disk cached index files. Otherwise, fetch and cache." + ), ) def __attrs_post_init__(self): @@ -1790,6 +1796,7 @@ def from_url(cls, url=ABOUT_BASE_URL, _LINKS_REPO={}, use_cached_index=False): _LINKS_REPO[url] = cls(url=url, use_cached_index=use_cached_index) return _LINKS_REPO[url] + ################################################################################ # Globals for remote repos to be lazily created and cached on first use for the # life of the session together with some convenience functions. @@ -1803,6 +1810,7 @@ def get_local_packages(directory=THIRDPARTY_DIR): """ return list(PypiPackage.packages_from_dir(directory=directory)) + ################################################################################ # # Basic file and URL-based operations using a persistent file-based Cache @@ -1953,6 +1961,7 @@ def fetch_and_save( fo.write(content) return content + ################################################################################ # # Functions to update or fetch ABOUT and license files @@ -2051,7 +2060,9 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) + local_dist.fetch_license_files( + dest_dir=dest_dir, use_cached_index=use_cached_index + ) continue # lets try to fetch remotely @@ -2089,7 +2100,9 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) + local_dist.fetch_license_files( + dest_dir=dest_dir, use_cached_index=use_cached_index + ) continue # try to get data from pkginfo (no license though) @@ -2107,6 +2120,7 @@ def get_other_dists(_package, _dist): lic_errs = "\n".join(lic_errs) print(f"Failed to fetch some licenses:: {lic_errs}") + ################################################################################ # # Functions to build new Python wheels including native on multiple OSes @@ -2210,6 +2224,7 @@ def download_wheels_with_pip( downloaded = existing ^ set(os.listdir(dest_dir)) return sorted(downloaded), error + ################################################################################ # # Functions to check for problems From d35d4feebe586a4218a8d421b6ca55a080291272 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 10 May 2022 14:27:01 +0200 Subject: [PATCH 37/41] Only use PyPI for downloads This is much faster Signed-off-by: Philippe Ombredanne --- configure | 3 +-- configure.bat | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/configure b/configure index d1b4fda..a52f539 100755 --- a/configure +++ b/configure @@ -54,11 +54,10 @@ CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin ################################ # Thirdparty package locations and index handling -# Find packages from the local thirdparty directory or from thirdparty.aboutcode.org +# Find packages from the local thirdparty directory if [ -d "$CFG_ROOT_DIR/thirdparty" ]; then PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty" fi -PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS --find-links https://thirdparty.aboutcode.org/pypi/simple/links.html" ################################ diff --git a/configure.bat b/configure.bat index 487e78a..41547cc 100644 --- a/configure.bat +++ b/configure.bat @@ -52,11 +52,10 @@ set "CFG_BIN_DIR=%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts" @rem ################################ @rem # Thirdparty package locations and index handling -@rem # Find packages from the local thirdparty directory or from thirdparty.aboutcode.org +@rem # Find packages from the local thirdparty directory if exist "%CFG_ROOT_DIR%\thirdparty" ( set PIP_EXTRA_ARGS=--find-links "%CFG_ROOT_DIR%\thirdparty" ) -set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% --find-links https://thirdparty.aboutcode.org/pypi/simple/links.html" @rem ################################ @@ -69,7 +68,6 @@ if not defined CFG_QUIET ( @rem ################################ @rem # Main command line entry point set "CFG_REQUIREMENTS=%REQUIREMENTS%" -set "NO_INDEX=--no-index" :again if not "%1" == "" ( From 453a22133de2b992bd4165a8c5fd199eb967a34e Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 10 May 2022 15:00:54 +0200 Subject: [PATCH 38/41] Add new dpkg version tests Signed-off-by: Philippe Ombredanne --- tests/dpkg-tests/README.rst | 10 + tests/dpkg-tests/gpl-2.0-plus.LICENSE | 352 ++++++++++++++++++ tests/dpkg-tests/test_version_dpkg.py | 248 ++++++++++++ tests/dpkg-tests/test_version_dpkg.py.ABOUT | 8 + .../dpkg-tests/test_version_python_debian.py | 92 +++++ .../test_version_python_debian.py.ABOUT | 11 + 6 files changed, 721 insertions(+) create mode 100644 tests/dpkg-tests/README.rst create mode 100644 tests/dpkg-tests/gpl-2.0-plus.LICENSE create mode 100644 tests/dpkg-tests/test_version_dpkg.py create mode 100644 tests/dpkg-tests/test_version_dpkg.py.ABOUT create mode 100644 tests/dpkg-tests/test_version_python_debian.py create mode 100644 tests/dpkg-tests/test_version_python_debian.py.ABOUT diff --git a/tests/dpkg-tests/README.rst b/tests/dpkg-tests/README.rst new file mode 100644 index 0000000..5b6fdd2 --- /dev/null +++ b/tests/dpkg-tests/README.rst @@ -0,0 +1,10 @@ +debian_inspector: dpkg tests +============================ + + +These tests are based on a heavily modified code from a subset of Debian's +python-debian and dpkg test suites are used to validate the version parsing and +comparison in the test suite. +See https://salsa.debian.org/python-debian-team/python-debian and +https://salsa.debian.org/dpkg-team/dpkg + diff --git a/tests/dpkg-tests/gpl-2.0-plus.LICENSE b/tests/dpkg-tests/gpl-2.0-plus.LICENSE new file mode 100644 index 0000000..ed62276 --- /dev/null +++ b/tests/dpkg-tests/gpl-2.0-plus.LICENSE @@ -0,0 +1,352 @@ +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 51 Franklin +Street, Fifth Floor, Boston, MA 02110-1301, USA. + + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/tests/dpkg-tests/test_version_dpkg.py b/tests/dpkg-tests/test_version_dpkg.py new file mode 100644 index 0000000..a4cc839 --- /dev/null +++ b/tests/dpkg-tests/test_version_dpkg.py @@ -0,0 +1,248 @@ +#!/usr/bin/python + +# SPDX-License-Identifier: GPL-2.0-or-later +# + +# libdpkg - Debian packaging suite library routines +# t-version.c - test version handling +# +# Copyright © 2009-2014 Guillem Jover +# +# This is free software you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation either version 2 of the License, or +# (at your option) any later version. +# +# This is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +from unittest import TestCase + +from debian_inspector import version +from unittest.case import expectedFailure + +DPKG_VERSION_OBJECT = version.Version +dpkg_version_compare = version.compare_versions +dpkg_version_relate = version.eval_constraint +parseversion = version.Version.from_string + + +class VersionTests(TestCase): + + def test_version_compare(self): + a = DPKG_VERSION_OBJECT(0, "1", "1") + b = DPKG_VERSION_OBJECT(0, "2", "1") + assert not dpkg_version_compare(a, b) == 0 + + a = DPKG_VERSION_OBJECT(0, "1", "1") + b = DPKG_VERSION_OBJECT(0, "1", "2") + assert not dpkg_version_compare(a, b) == 0 + + # Test for version equality. + a = b = DPKG_VERSION_OBJECT(0, "0", "0") + assert dpkg_version_compare(a, b) == 0 + + a = DPKG_VERSION_OBJECT(0, "0", "00") + b = DPKG_VERSION_OBJECT(0, "00", "0") + assert dpkg_version_compare(a, b) == 0 + + a = b = DPKG_VERSION_OBJECT(1, "2", "3") + assert dpkg_version_compare(a, b) == 0 + + # Test for epoch difference. + a = DPKG_VERSION_OBJECT(0, "0", "0") + b = DPKG_VERSION_OBJECT(1, "0", "0") + assert dpkg_version_compare(a, b) < 0 + assert dpkg_version_compare(b, a) > 0 + + # Test for version component difference. + a = DPKG_VERSION_OBJECT(0, "a", "0") + b = DPKG_VERSION_OBJECT(0, "b", "0") + assert dpkg_version_compare(a, b) < 0 + assert dpkg_version_compare(b, a) > 0 + + # Test for revision component difference. + a = DPKG_VERSION_OBJECT(0, "0", "a") + b = DPKG_VERSION_OBJECT(0, "0", "b") + assert dpkg_version_compare(a, b) < 0 + assert dpkg_version_compare(b, a) > 0 + + # FIXME: Complete. + + def test_version_relate(self): + + a = DPKG_VERSION_OBJECT(0, "1", "1") + b = DPKG_VERSION_OBJECT(0, "1", "1") + assert dpkg_version_relate(a, '=', b) + assert not dpkg_version_relate(a, '<<', b) + assert dpkg_version_relate(a, '<=', b) + assert not dpkg_version_relate(a, '>>', b) + assert dpkg_version_relate(a, '>=', b) + + a = DPKG_VERSION_OBJECT(0, "1", "1") + b = DPKG_VERSION_OBJECT(0, "2", "1") + assert not dpkg_version_relate(a, '=', b) + assert dpkg_version_relate(a, '<<', b) + assert dpkg_version_relate(a, '<=', b) + assert not dpkg_version_relate(a, '>>', b) + assert not dpkg_version_relate(a, '>=', b) + + a = DPKG_VERSION_OBJECT(0, "2", "1") + b = DPKG_VERSION_OBJECT(0, "1", "1") + assert not dpkg_version_relate(a, '=', b) + assert not dpkg_version_relate(a, '<<', b) + assert not dpkg_version_relate(a, '<=', b) + assert dpkg_version_relate(a, '>>', b) + assert dpkg_version_relate(a, '>=', b) + + def test_version_parse(self): + b = DPKG_VERSION_OBJECT(0, "0", "") + + a = parseversion("0") + assert dpkg_version_compare(a, b) == 0 + + a = parseversion("0:0") + assert dpkg_version_compare(a, b) == 0 + + b = DPKG_VERSION_OBJECT(0, "0", "0") + a = parseversion("0:0-0") + assert dpkg_version_compare(a, b) == 0 + + b = DPKG_VERSION_OBJECT(0, "0.0", "0.0") + a = parseversion("0:0.0-0.0") + assert dpkg_version_compare(a, b) == 0 + + # Test epoched versions. + b = DPKG_VERSION_OBJECT(1, "0", "") + a = parseversion("1:0") + assert dpkg_version_compare(a, b) == 0 + + b = DPKG_VERSION_OBJECT(5, "1", "") + a = parseversion("5:1") + assert dpkg_version_compare(a, b) == 0 + + # Test multiple hyphens. + b = DPKG_VERSION_OBJECT(0, "0-0", "0") + a = parseversion("0:0-0-0") + assert dpkg_version_compare(a, b) == 0 + + b = DPKG_VERSION_OBJECT(0, "0-0-0", "0") + a = parseversion("0:0-0-0-0") + assert dpkg_version_compare(a, b) == 0 + + @expectedFailure + def test_version_parse_fails_with_multiple_colons_while_dpkg_pass1(self): + # Test multiple colons. + b = DPKG_VERSION_OBJECT(0, "0:0", "0") + a = parseversion("0:0:0-0") + assert dpkg_version_compare(a, b) == 0 + + @expectedFailure + def test_version_parse_fails_with_multiple_colons_while_dpkg_pass2(self): + b = DPKG_VERSION_OBJECT(0, "0:0:0", "0") + a = parseversion("0:0:0:0-0") + assert dpkg_version_compare(a, b) == 0 + + @expectedFailure + def test_version_parse_fails_with_multiple_colons_while_dpkg_pass3(self): + # Test multiple hyphens and colons. + b = DPKG_VERSION_OBJECT(0, "0:0-0", "0") + a = parseversion("0:0:0-0-0") + assert dpkg_version_compare(a, b) == 0 + + @expectedFailure + def test_version_parse_fails_with_multiple_colons_while_dpkg_pass4(self): + b = DPKG_VERSION_OBJECT(0, "0-0:0", "0") + a = parseversion("0:0-0:0-0") + assert dpkg_version_compare(a, b) == 0 + + @expectedFailure + def test_version_parse_fails_with_multiple_colons_while_dpkg_pass5(self): + # Test valid characters in upstream version. + b = DPKG_VERSION_OBJECT(0, "09azAZ.-+~:", "0") + a = parseversion("0:09azAZ.-+~:-0") + assert dpkg_version_compare(a, b) == 0 + + @expectedFailure + def test_version_parse_handles_invalid_chars_in_dpkg1(self): + # Test valid characters in revision. + b = DPKG_VERSION_OBJECT(0, "0", "azAZ09.+~") + a = parseversion("0:0-azAZ09.+~") + assert dpkg_version_compare(a, b) == 0 + + def test_version_parse_handles_valid_chars_in_revision(self): + # Test valid characters in revision. + b = DPKG_VERSION_OBJECT(0, "0", "azA.+~Z09") + a = parseversion("0:0-azA.+~Z09") + assert dpkg_version_compare(a, b) == 0 + + def test_version_parse_handles_invalid_chars_in_dpkg2(self): + # Test version with leading and trailing spaces. + b = DPKG_VERSION_OBJECT(0, "0", "1") + a = parseversion(" 0:0-1") + assert dpkg_version_compare(a, b) == 0 + a = parseversion("0:0-1 ") + assert dpkg_version_compare(a, b) == 0 + a = parseversion(" 0:0-1 ") + assert dpkg_version_compare(a, b) == 0 + + def test_version_parse_raise_exception_on_multiple_colons(self): + self.assertRaises(ValueError, parseversion, "0:0-0:0-0") + self.assertRaises(ValueError, parseversion, "0:0:0:0-0") + self.assertRaises(ValueError, parseversion, "0:0:0-0-0") + self.assertRaises(ValueError, parseversion, "0:0-0:0-0") + self.assertRaises(ValueError, parseversion, "0:09azAZ.-+~:-0") + self.assertRaises(ValueError, parseversion, "0:0-azAZ09.+~") + + def test_version_parse_exceptions(self): + + # Test empty version. + self.assertRaises(ValueError, parseversion, "") + self.assertRaises(ValueError, parseversion, " ") + + # Test empty upstream version after epoch. + self.assertRaises(ValueError, parseversion, "0:") + + # Test empty epoch in version. + self.assertRaises(ValueError, parseversion, ":1.0") + + # Test empty revision in version. + self.assertRaises(ValueError, parseversion, "1.0-") + + # Test version with embedded spaces. + self.assertRaises(ValueError, parseversion, "0:0 0-1") + + # Test version with negative epoch. + self.assertRaises(ValueError, parseversion, "-1:0-1") + + # Test invalid characters in epoch. + self.assertRaises(ValueError, parseversion, "a:0-0") + self.assertRaises(ValueError, parseversion, "A:0-0") + + # Test version with huge epoch. + # self.assertRaises(ValueError, parseversion, "999999999999999999999999:0-1") + + # Test invalid empty upstream version. + self.assertRaises(ValueError, parseversion, "-0") + self.assertRaises(ValueError, parseversion, "0:-0") + + # Test upstream version not starting with a digit + self.assertRaises(ValueError, parseversion, "0:abc3-0") + + # Test invalid characters in upstream version. + for p in "!#@$%&/|\\<>()[]{},_=*^'": + verstr = '0:0a' + p + '-0' + self.assertRaises(ValueError, parseversion, verstr) + + # Test invalid characters in revision. + self.assertRaises(ValueError, parseversion, "0:0-0:0") + + for p in "!#@$%&/|\\<>()[]{}:,_=*^'": + verstr = '0:0-0' + p + self.assertRaises(ValueError, parseversion, verstr) diff --git a/tests/dpkg-tests/test_version_dpkg.py.ABOUT b/tests/dpkg-tests/test_version_dpkg.py.ABOUT new file mode 100644 index 0000000..35a3c8b --- /dev/null +++ b/tests/dpkg-tests/test_version_dpkg.py.ABOUT @@ -0,0 +1,8 @@ +about_resource: test_version_dpkg.py +name: dpkg +license_expression: gpl-2.0-plus +download_url: https://salsa.debian.org/dpkg-team/dpkg/-/blob/55cdc52e9a9bccc7ba59087917f5cf7d88e312e4/lib/dpkg/t/t-version.c +copyright:Copyright (c) 2009-2014 Guillem Jover +package_url: pkg:generic/dpkg@@1.20.7.1#lib/dpkg/t/t-version.c +notes: this subset of dpkg tests has been heavily modified and ported from C to Python + to tests version comparison and parsing \ No newline at end of file diff --git a/tests/dpkg-tests/test_version_python_debian.py b/tests/dpkg-tests/test_version_python_debian.py new file mode 100644 index 0000000..01d6326 --- /dev/null +++ b/tests/dpkg-tests/test_version_python_debian.py @@ -0,0 +1,92 @@ +#!/usr/bin/python + +# SPDX-License-Identifier: GPL-2.0-or-later +# +# Copyright (C) 2005 Florian Weimer +# Copyright (C) 2006-2007 James Westby +# Copyright (C) 2010 John Wright +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +# this subset of Python debian tests has been modified to tests version +# comparison and parsing + +from unittest import TestCase + +from debian_inspector import version + + +def check_Version_from_string(version_string, epoch, upstream, revision): + v = version.Version.from_string(version_string) + result = v.epoch, v.upstream, v.revision + expected = epoch, upstream, revision + assert result == expected + + +def check_compare_versions(version1, operator, version2): + expected_ops = { + '==': 0, + '>': 1, + '<':-1, + } + expected = expected_ops[operator] + compared = version.compare_versions(version1, version2) + assert compared == expected + + +class VersionTests(TestCase): + + def test_Version_from_string(self): + check_Version_from_string('1:1.4.1-1', 1, '1.4.1', '1') + check_Version_from_string('7.1.ds-1', 0, '7.1.ds', '1') + check_Version_from_string('10.11.1.3-2', 0, '10.11.1.3', '2') + check_Version_from_string('4.0.1.3.dfsg.1-2', 0, '4.0.1.3.dfsg.1', '2') + check_Version_from_string('0.4.23debian1', 0, '0.4.23debian1', '0') + check_Version_from_string('1.2.10+cvs20060429-1', 0, '1.2.10+cvs20060429', '1') + check_Version_from_string('0.2.0-1+b1', 0, '0.2.0', '1+b1') + check_Version_from_string('4.3.90.1svn-r21976-1', 0, '4.3.90.1svn-r21976', '1') + check_Version_from_string('1.5+E-14', 0, '1.5+E', '14') + check_Version_from_string('20060611-0.0', 0, '20060611', '0.0') + check_Version_from_string('0.52.2-5.1', 0, '0.52.2', '5.1') + check_Version_from_string('7.0-035+1', 0, '7.0', '035+1') + check_Version_from_string('1.1.0+cvs20060620-1+2.6.15-8', 0, '1.1.0+cvs20060620-1+2.6.15', '8') + check_Version_from_string('1.1.0+cvs20060620-1+1.0', 0, '1.1.0+cvs20060620', '1+1.0') + check_Version_from_string('4.2.0a+stable-2sarge1', 0, '4.2.0a+stable', '2sarge1') + check_Version_from_string('1.8RC4b', 0, '1.8RC4b', '0') + check_Version_from_string('0.9~rc1-1', 0, '0.9~rc1', '1') + check_Version_from_string('2:1.0.4+svn26-1ubuntu1', 2, '1.0.4+svn26', '1ubuntu1') + check_Version_from_string('2:1.0.4~rc2-1', 2, '1.0.4~rc2', '1') + self.assertRaises(ValueError, version.Version.from_string, 'a1:1.8.8-070403-1~priv1') + + def test_compare_versions(self): + check_compare_versions('1.0', '<', '1.1') + check_compare_versions('1.2', '<', '1.11') + check_compare_versions('1.0-0.1', '<', '1.1') + check_compare_versions('1.0-0.1', '<', '1.0-1') + check_compare_versions('1.0', '==', '1.0') + check_compare_versions('1.0-0.1', '==', '1.0-0.1') + check_compare_versions('1:1.0-0.1', '==', '1:1.0-0.1') + check_compare_versions('1:1.0', '==', '1:1.0') + check_compare_versions('1.0-0.1', '<', '1.0-1') + check_compare_versions('1.0final-5sarge1', '>', '1.0final-5') + check_compare_versions('1.0final-5', '>', '1.0a7-2') + check_compare_versions('0.9.2-5', '<', '0.9.2+cvs.1.0.dev.2004.07.28-1.5') + check_compare_versions('1:500', '<', '1:5000') + check_compare_versions('100:500', '>', '11:5000') + check_compare_versions('1.0.4-2', '>', '1.0pre7-2') + check_compare_versions('1.5~rc1', '<', '1.5') + check_compare_versions('1.5~rc1', '<', '1.5+b1') + check_compare_versions('1.5~rc1', '<', '1.5~rc2') + check_compare_versions('1.5~rc1', '>', '1.5~dev0') diff --git a/tests/dpkg-tests/test_version_python_debian.py.ABOUT b/tests/dpkg-tests/test_version_python_debian.py.ABOUT new file mode 100644 index 0000000..0e373a3 --- /dev/null +++ b/tests/dpkg-tests/test_version_python_debian.py.ABOUT @@ -0,0 +1,11 @@ +about_resource: test_version_python_debian.py +name: python-debian +license_expression: gpl-2.0-plus +download_url: https://salsa.debian.org/python-debian-team/python-debian/-/raw/b3e0d0f3b77e5107a5b627faf0c459bd3b47b1f1/lib/debian/tests/test_debian_support.py +copyright: | + Copyright (C) 2005 Florian Weimer + Copyright (C) 2006-2007 James Westby + Copyright (C) 2010 John Wright +package_url: pkg:pypi/python-debian@0.1.39#lib/debian/tests/test_debian_support.py +version: b3e0d0f3b77e5107a5b627faf0c459bd3b47b1f1 +notes: this subset of Python debian tests has been modified to tests version comparison and parsing \ No newline at end of file From e19397330e15d41fe9059c9ab1352db3986b84d3 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 10 May 2022 15:09:23 +0200 Subject: [PATCH 39/41] Sync config with SCTK Signed-off-by: Philippe Ombredanne --- .travis.yml | 22 ------------ CODE_OF_CONDUCT.rst | 86 ++++++++++++++++++++++++++++++++++++++++++++ MANIFEST.in | 2 +- README.rst | 7 ++-- requirements-dev.txt | 24 +++++++++++++ requirements.txt | 80 +++++++++++++++++++++++++++++++++++++++++ setup.cfg | 8 +++-- 7 files changed, 202 insertions(+), 27 deletions(-) delete mode 100644 .travis.yml create mode 100644 CODE_OF_CONDUCT.rst diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index ea48ceb..0000000 --- a/.travis.yml +++ /dev/null @@ -1,22 +0,0 @@ -# This is a skeleton Travis CI config file that provides a starting point for adding CI -# to a Python project. Since we primarily develop in python3, this skeleton config file -# will be specific to that language. -# -# See https://config.travis-ci.com/ for a full list of configuration options. - -os: linux - -dist: xenial - -language: python -python: - - "3.6" - - "3.7" - - "3.8" - - "3.9" - -# Scripts to run at install stage -install: ./configure --dev - -# Scripts to run at script stage -script: venv/bin/pytest diff --git a/CODE_OF_CONDUCT.rst b/CODE_OF_CONDUCT.rst new file mode 100644 index 0000000..590ba19 --- /dev/null +++ b/CODE_OF_CONDUCT.rst @@ -0,0 +1,86 @@ +Contributor Covenant Code of Conduct +==================================== + +Our Pledge +---------- + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our +project and our community a harassment-free experience for everyone, +regardless of age, body size, disability, ethnicity, gender identity and +expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +Our Standards +------------- + +Examples of behavior that contributes to creating a positive environment +include: + +- Using welcoming and inclusive language +- Being respectful of differing viewpoints and experiences +- Gracefully accepting constructive criticism +- Focusing on what is best for the community +- Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +- The use of sexualized language or imagery and unwelcome sexual + attention or advances +- Trolling, insulting/derogatory comments, and personal or political + attacks +- Public or private harassment +- Publishing others’ private information, such as a physical or + electronic address, without explicit permission +- Other conduct which could reasonably be considered inappropriate in a + professional setting + +Our Responsibilities +-------------------- + +Project maintainers are responsible for clarifying the standards of +acceptable behavior and are expected to take appropriate and fair +corrective action in response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, +or reject comments, commits, code, wiki edits, issues, and other +contributions that are not aligned to this Code of Conduct, or to ban +temporarily or permanently any contributor for other behaviors that they +deem inappropriate, threatening, offensive, or harmful. + +Scope +----- + +This Code of Conduct applies both within project spaces and in public +spaces when an individual is representing the project or its community. +Examples of representing a project or community include using an +official project e-mail address, posting via an official social media +account, or acting as an appointed representative at an online or +offline event. Representation of a project may be further defined and +clarified by project maintainers. + +Enforcement +----------- + +Instances of abusive, harassing, or otherwise unacceptable behavior may +be reported by contacting the project team at pombredanne@gmail.com +or on the Gitter chat channel at https://gitter.im/aboutcode-org/discuss . +All complaints will be reviewed and investigated and will result in a +response that is deemed necessary and appropriate to the circumstances. +The project team is obligated to maintain confidentiality with regard to +the reporter of an incident. Further details of specific enforcement +policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in +good faith may face temporary or permanent repercussions as determined +by other members of the project’s leadership. + +Attribution +----------- + +This Code of Conduct is adapted from the `Contributor Covenant`_ , +version 1.4, available at +https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +.. _Contributor Covenant: https://www.contributor-covenant.org diff --git a/MANIFEST.in b/MANIFEST.in index ef3721e..8424cbe 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -9,7 +9,7 @@ include *.rst include setup.* include configure* include requirements* -include .git* +include .giti* global-exclude *.py[co] __pycache__ *.*~ diff --git a/README.rst b/README.rst index 2dc7116..8832ee3 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,8 @@ -debian_inspector: Utilities to parse Debian package, copyright and control files -================================================================================ +================================= +debian_inspector +================================= + +Utilities to parse Debian package, copyright and control files The Python package `debian_inspector` is a collection of utilities to parse various Debian package manifests, machine readable copyright and control files diff --git a/requirements-dev.txt b/requirements-dev.txt index e69de29..fe92ed8 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -0,0 +1,24 @@ +aboutcode-toolkit==7.0.1 +bleach==4.1.0 +build==0.7.0 +commonmark==0.9.1 +docutils==0.18.1 +et-xmlfile==1.1.0 +execnet==1.9.0 +iniconfig==1.1.1 +jeepney==0.7.1 +keyring==23.4.1 +openpyxl==3.0.9 +pep517==0.12.0 +pkginfo==1.8.2 +py==1.11.0 +pytest==7.0.1 +pytest-forked==1.4.0 +pytest-xdist==2.5.0 +readme-renderer==34.0 +requests-toolbelt==0.9.1 +rfc3986==1.5.0 +rich==12.3.0 +secretstorage==3.3.2 +tomli==1.2.3 +twine==3.8.0 diff --git a/requirements.txt b/requirements.txt index e69de29..c099710 100644 --- a/requirements.txt +++ b/requirements.txt @@ -0,0 +1,80 @@ +attrs==21.4.0 +banal==1.0.6 +beautifulsoup4==4.11.1 +binaryornot==0.4.4 +boolean.py==3.8 +certifi==2021.10.8 +cffi==1.15.0 +chardet==4.0.0 +charset-normalizer==2.0.12 +click==8.0.4 +colorama==0.4.4 +commoncode==30.2.0 +construct==2.10.68 +container-inspector==31.0.0 +cryptography==36.0.2 +debian-inspector==30.0.0 +dockerfile-parse==1.2.0 +dparse2==0.6.1 +extractcode==31.0.0 +extractcode-7z==16.5.210531 +extractcode-libarchive==3.5.1.210531 +fasteners==0.17.3 +fingerprints==1.0.3 +ftfy==6.0.3 +future==0.18.2 +gemfileparser==0.8.0 +html5lib==1.1 +idna==3.3 +importlib-metadata==4.8.3 +inflection==0.5.1 +intbitset==3.0.1 +isodate==0.6.1 +jaraco.functools==3.4.0 +javaproperties==0.8.1 +Jinja2==3.0.3 +jsonstreams==0.6.0 +license-expression==21.6.14 +lxml==4.8.0 +MarkupSafe==2.0.1 +more-itertools==8.13.0 +normality==2.3.3 +packagedcode-msitools==0.101.210706 +packageurl-python==0.9.9 +packaging==21.3 +parameter-expansion-patched==0.3.1 +patch==1.16 +pdfminer-six==20220506 +pefile==2021.9.3 +pip-requirements-parser==31.2.0 +pkginfo2==30.0.0 +pluggy==1.0.0 +plugincode==30.0.0 +ply==3.11 +publicsuffix2==2.20191221 +pyahocorasick==2.0.0b1 +pycparser==2.21 +pygmars==0.7.0 +Pygments==2.12.0 +pymaven-patch==0.3.0 +pyparsing==3.0.8 +pytz==2022.1 +PyYAML==6.0 +rdflib==5.0.0 +regipy==2.3.1 +requests==2.27.1 +rpm-inspector-rpm==4.16.1.3.210404 +saneyaml==0.5.2 +six==1.16.0 +soupsieve==2.3.1 +spdx-tools==0.7.0a3 +text-unidecode==1.3 +toml==0.10.2 +typecode==30.0.0 +typecode-libmagic==5.39.210531 +urllib3==1.26.9 +urlpy==0.5 +wcwidth==0.2.5 +webencodings==0.5.1 +xmltodict==0.12.0 +zipp==3.6.0 diff --git a/setup.cfg b/setup.cfg index c6c5207..3742e5f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,6 +16,11 @@ classifiers = Intended Audience :: Developers Programming Language :: Python :: 3 Programming Language :: Python :: 3 :: Only + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 Topic :: Software Development Topic :: Utilities @@ -56,18 +61,17 @@ install_requires = attrs >=19.2, !=20.1.0 - [options.packages.find] where = src [options.extras_require] testing = - commoncode pytest >= 6, != 7.0.0 pytest-xdist >= 2 aboutcode-toolkit >= 6.0.0 black + commoncode docs = Sphinx >= 3.3.1 From 93931ae604b998a7047f46a1f804e9143effaa3c Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Wed, 25 May 2022 12:33:55 -0700 Subject: [PATCH 40/41] Remove debian-inspector pinned dependency Signed-off-by: Jono Yang --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c099710..5498812 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,6 @@ commoncode==30.2.0 construct==2.10.68 container-inspector==31.0.0 cryptography==36.0.2 -debian-inspector==30.0.0 dockerfile-parse==1.2.0 dparse2==0.6.1 extractcode==31.0.0 From 4ba461f2d3f0d633238cacbdfbf3c1bb320dd247 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Wed, 25 May 2022 12:40:42 -0700 Subject: [PATCH 41/41] Update CHANGELOG.rst Signed-off-by: Jono Yang --- CHANGELOG.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 69be808..fda86a6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,13 @@ Changelog ========= +v30.0.1 - 2022-05-25 +------------------------ + +- Fix error when using ``get_license_detection_from_nameless_paragraph()`` on + Debian copyright files who named the license field as ``licence`` #25 + + v30.0.0 - 2021-09-07 ------------------------ @@ -57,7 +64,7 @@ v0.9.6 - 2020-07-07 ------------------- - Support parsing Alpine installed db. This is a debian-like file but the keys - are case-sensitive + are case-sensitive v0.9.5 - 2020-06-09