Merge remote-tracking branch 'skeleton/main' into update-skeleton-files

.github/workflows/docs-ci.yml

@@ -13,10 +13,10 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 

.github/workflows/pypi-release.yml

@@ -1,27 +1,83 @@
-name: Release library as a PyPI wheel and sdist on GH release creation
+name: Create library release archives, create a GH release and publish PyPI wheel and sdist on tag in main branch
+
+
+# This is executed automatically on a tag in the main branch
+
+# Summary of the steps:
+# - build wheels and sdist
+# - upload wheels and sdist to PyPI
+# - create gh-release and upload wheels and dists there
+# TODO: smoke test wheels and sdist
+# TODO: add changelog to release text body
+
+# WARNING: this is designed only for packages building as pure Python wheels
 
 on:
-  release:
-    types: [created]
+  workflow_dispatch:
+  push:
+    tags:
+      - "v*.*.*"
 
 jobs:
-  build-and-publish-to-pypi:
+  build-pypi-distribs:
     name: Build and publish library to PyPI
     runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
+      - name: Install pypa/build
+        run: python -m pip install build --user
+
+      - name: Build a binary wheel and a source tarball
+        run: python -m build --sdist --wheel --outdir dist/
+
+      - name: Upload built archives
+        uses: actions/upload-artifact@v3
+        with:
+          name: pypi_archives
+          path: dist/*
+
+
+  create-gh-release:
+    name: Create GH release
+    needs:
+      - build-pypi-distribs
+    runs-on: ubuntu-20.04
+
+    steps:
+      - name: Download built archives
+        uses: actions/download-artifact@v3
+        with:
+          name: pypi_archives
+          path: dist
+
+      - name: Create GH release
+        uses: softprops/action-gh-release@v1
+        with:
+          draft: true
+          files: dist/*
+
+
+  create-pypi-release:
+    name: Create PyPI release
+    needs:
+      - create-gh-release
+    runs-on: ubuntu-20.04
+
     steps:
-     - uses: actions/checkout@master
-     - name: Set up Python
-       uses: actions/setup-python@v1
-       with:
-         python-version: 3.9
-     - name: Install pypa/build
-       run: python -m pip install build --user
-     - name: Build a binary wheel and a source tarball
-       run: python -m build --sdist --wheel --outdir dist/
-        .
-     - name: Publish distribution  to PyPI
-       if: startsWith(github.ref, 'refs/tags')
-       uses: pypa/gh-action-pypi-publish@master
-       with:
-         password: ${{ secrets.PYPI_API_TOKEN }}
-
+      - name: Download built archives
+        uses: actions/download-artifact@v3
+        with:
+          name: pypi_archives
+          path: dist
+
+      - name: Publish to PyPI
+        if: startsWith(github.ref, 'refs/tags')
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}

.gitignore

@@ -2,6 +2,7 @@
 *.py[cod]
 
 # virtualenv and other misc bits
+/src/*.egg-info
 *.egg-info
 /dist
 /build

.readthedocs.yml

@@ -5,6 +5,17 @@
 # Required
 version: 2
 
+# Build in latest ubuntu/python
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
+# Build PDF & ePub
+formats:
+  - epub
+  - pdf
+
 # Where the Sphinx conf.py file is located
 sphinx:
    configuration: docs/source/conf.py

Makefile

@@ -0,0 +1,54 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/skeleton for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+# Python version can be specified with `$ PYTHON_EXE=python3.x make conf`
+PYTHON_EXE?=python3
+VENV=venv
+ACTIVATE?=. ${VENV}/bin/activate;
+
+dev:
+	@echo "-> Configure the development envt."
+	./configure --dev
+
+isort:
+	@echo "-> Apply isort changes to ensure proper imports ordering"
+	${VENV}/bin/isort --sl -l 100 src tests setup.py
+
+black:
+	@echo "-> Apply black code formatter"
+	${VENV}/bin/black -l 100 src tests setup.py
+
+doc8:
+	@echo "-> Run doc8 validation"
+	@${ACTIVATE} doc8 --max-line-length 100 --ignore-path docs/_build/ --quiet docs/
+
+valid: isort black
+
+check:
+	@echo "-> Run pycodestyle (PEP8) validation"
+	@${ACTIVATE} pycodestyle --max-line-length=100 --exclude=.eggs,venv,lib,thirdparty,docs,migrations,settings.py,.cache .
+	@echo "-> Run isort imports ordering validation"
+	@${ACTIVATE} isort --sl --check-only -l 100 setup.py src tests . 
+	@echo "-> Run black validation"
+	@${ACTIVATE} black --check --check -l 100 src tests setup.py
+
+clean:
+	@echo "-> Clean the Python env"
+	./configure --clean
+
+test:
+	@echo "-> Run the test suite"
+	${VENV}/bin/pytest -vvs
+
+docs:
+	rm -rf docs/_build/
+	@${ACTIVATE} sphinx-build docs/ docs/_build/
+
+.PHONY: conf dev check valid black isort clean test docs

README.rst

@@ -1,8 +1,8 @@
-FetchCode: Utilities to fetch code from purls, URLs and version control repos. 
+FetchCode: Utilities to fetch code from purls, URLs and version control repos.
 ================================================================================
 
 FetchCode is a library to reliably fetch code via HTTP, FTP and version control
-systems. It can work using plain HTTP and FTP URLs, as well as 
+systems. It can work using plain HTTP and FTP URLs, as well as
 `Package URLs <https://github.com/package-url>`_ and version control (VCS) URLs
 as used in Python pip and as specified in `SPDX Package Download Location
 <https://spdx.github.io/spdx-spec/3-package-information/#37-package-download-location>`_

azure-pipelines.yml

@@ -9,48 +9,56 @@ jobs:
 
     - template: etc/ci/azure-posix.yml
       parameters:
-          job_name: ubuntu18_cpython
-          image_name: ubuntu-18.04
-          python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10']
+          job_name: ubuntu20_cpython
+          image_name: ubuntu-20.04
+          python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11']
           test_suites:
               all: venv/bin/pytest -n 2 -vvs
 
     - template: etc/ci/azure-posix.yml
       parameters:
-          job_name: ubuntu20_cpython
-          image_name: ubuntu-20.04
-          python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10']
+          job_name: ubuntu22_cpython
+          image_name: ubuntu-22.04
+          python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11']
+          test_suites:
+              all: venv/bin/pytest -n 2 -vvs
+
+    - template: etc/ci/azure-posix.yml
+      parameters:
+          job_name: macos11_cpython
+          image_name: macOS-11
+          python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11']
           test_suites:
               all: venv/bin/pytest -n 2 -vvs
 
     - template: etc/ci/azure-posix.yml
       parameters:
           job_name: macos12_cpython
-          image_name: macos-12
-          python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10']
+          image_name: macOS-12
+          python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11']
           test_suites:
               all: venv/bin/pytest -n 2 -vvs
 
     - template: etc/ci/azure-posix.yml
       parameters:
-          job_name: macos11_cpython
-          image_name: macos-11
-          python_versions: ['3.7', '3.8', '3.9', '3.10']
+          job_name: macos13_cpython
+          image_name: macOS-13
+          python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11']
           test_suites:
               all: venv/bin/pytest -n 2 -vvs
 
     - template: etc/ci/azure-win.yml
       parameters:
           job_name: win2019_cpython
           image_name: windows-2019
-          python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10']
+          python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11']
           test_suites:
               all: venv\Scripts\pytest -n 2 -vvs
 
     - template: etc/ci/azure-win.yml
       parameters:
           job_name: win2022_cpython
           image_name: windows-2022
-          python_versions: ['3.7', '3.8', '3.9', '3.10']
+          python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11']
           test_suites:
               all: venv\Scripts\pytest -n 2 -vvs

configure

@@ -36,7 +36,7 @@ DOCS_REQUIREMENTS="--editable .[docs] --constraint requirements.txt"
 VIRTUALENV_DIR=venv
 
 # Cleanable files and directories to delete with the --clean option
-CLEANABLE="build venv"
+CLEANABLE="build dist venv .cache .eggs"
 
 # extra  arguments passed to pip
 PIP_EXTRA_ARGS=" "
@@ -52,13 +52,20 @@ CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin
 
 
+################################
+# Install with or without and index. With "--no-index" this is using only local wheels
+# This is an offline mode with no index and no network operations
+# NO_INDEX="--no-index "
+NO_INDEX=""
+
+
 ################################
 # Thirdparty package locations and index handling
-# Find packages from the local thirdparty directory or from thirdparty.aboutcode.org
-if [ -d "$CFG_ROOT_DIR/thirdparty" ]; then
-    PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty"
+# Find packages from the local thirdparty directory if present
+THIRDPARDIR=$CFG_ROOT_DIR/thirdparty
+if [[ "$(echo $THIRDPARDIR/*.whl)x" != "$THIRDPARDIR/*.whlx" ]]; then
+    PIP_EXTRA_ARGS="$NO_INDEX --find-links $THIRDPARDIR"
 fi
-PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS --find-links https://thirdparty.aboutcode.org/pypi/simple/links.html"
 
 
 ################################
@@ -183,6 +190,7 @@ while getopts :-: optchar; do
     esac
 done
 
+
 PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS"
 
 find_python

configure.bat

@@ -34,7 +34,7 @@ set "DOCS_REQUIREMENTS=--editable .[docs] --constraint requirements.txt"
 set "VIRTUALENV_DIR=venv"
 
 @rem # Cleanable files and directories to delete with the --clean option
-set "CLEANABLE=build venv"
+set "CLEANABLE=build dist venv .cache .eggs"
 
 @rem # extra  arguments passed to pip
 set "PIP_EXTRA_ARGS= "
@@ -52,11 +52,10 @@ set "CFG_BIN_DIR=%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts"
 
 @rem ################################
 @rem # Thirdparty package locations and index handling
-@rem # Find packages from the local thirdparty directory or from thirdparty.aboutcode.org
+@rem # Find packages from the local thirdparty directory
 if exist "%CFG_ROOT_DIR%\thirdparty" (
     set PIP_EXTRA_ARGS=--find-links "%CFG_ROOT_DIR%\thirdparty"
 )
-set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% --find-links https://thirdparty.aboutcode.org/pypi/simple/links.html"
 
 
 @rem ################################
@@ -69,7 +68,6 @@ if not defined CFG_QUIET (
 @rem ################################
 @rem # Main command line entry point
 set "CFG_REQUIREMENTS=%REQUIREMENTS%"
-set "NO_INDEX=--no-index"
 
 :again
 if not "%1" == "" (