Improve packaging #3459

.github/workflows/scancode-release.yml

@@ -20,11 +20,11 @@ on:
 permissions: {}
 jobs:
 
-  build_scancode_wheel_for_pypi:
+  build_scancode_wheels_for_pypi:
     permissions:
       contents: read # to fetch code (actions/checkout)
 
-    name: Build PyPI wheels
+    name: Build PyPI packages
     runs-on: ubuntu-22.04
 
     defaults:
@@ -34,7 +34,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-       pyver: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+       pyver: ["3.8"]
 
     steps:
       - uses: actions/checkout@v4
@@ -44,46 +44,14 @@ jobs:
         with:
           python-version: ${{ matrix.pyver }} 
 
-      - name: Install requirements then build main and mini wheel
-        run: etc/release/scancode-create-pypi-wheel.sh
-
-      - name: Collect built wheels for ${{ matrix.pyver }} 
-        uses: actions/upload-artifact@v3
-        with:
-          name: wheels-${{ matrix.pyver }}
-          path: dist/*.whl
-
-
-  build_scancode_sdist_for_pypi:
-    permissions:
-      contents: read # to fetch code (actions/checkout)
-
-    name: Build PyPI sdist archives
-    runs-on: ubuntu-22.04
-
-    defaults:
-      run:
-        shell: bash
-
-    strategy:
-      fail-fast: true
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
+      - name: Install requirements then build all PyPI packages
+        run: etc/release/scancode-create-pypi-wheel-and-sdist.sh
 
-      - name: Install requirements then build main and mini sdist
-        run: etc/release/scancode-create-pypi-sdist.sh
-
-      - name: Collect built sdist
+      - name: Collect built wheels and dist 
         uses: actions/upload-artifact@v3
         with:
-          name: sdists
-          path: dist/*.tar.gz
+          name: wheels
+          path: dist/*
 
 
   build_scancode_for_release_linux:
@@ -93,6 +61,9 @@ jobs:
     name: Build app Release for linux
     runs-on: ubuntu-22.04
 
+    needs:
+      - build_scancode_wheels_for_pypi
+
     defaults:
       run:
         shell: bash
@@ -110,6 +81,12 @@ jobs:
         with:
           python-version: ${{ matrix.pyver }} 
 
+      - name: Download wheels artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: wheels
+          path: dist
+
       - name: Build linux app archive
         run: etc/release/scancode-create-release-app-linux.sh ${{ matrix.pyver }}
 
@@ -127,6 +104,9 @@ jobs:
     name: Build app Release for mac
     runs-on: ubuntu-22.04
 
+    needs:
+      - build_scancode_wheels_for_pypi
+
     defaults:
       run:
         shell: bash
@@ -137,13 +117,20 @@ jobs:
        pyver: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
+
       - uses: actions/checkout@v4
 
       - name: Set up Python on ${{ matrix.pyver }}
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.pyver }} 
 
+      - name: Download wheels artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: wheels
+          path: dist
+
       - name: Build mac app archive
         run: etc/release/scancode-create-release-app-macos.sh ${{ matrix.pyver }}
 
@@ -161,6 +148,9 @@ jobs:
     name: Build app Release for windows
     runs-on: ubuntu-22.04
 
+    needs:
+      - build_scancode_wheels_for_pypi
+
     defaults:
       run:
         shell: bash
@@ -178,6 +168,12 @@ jobs:
         with:
           python-version: ${{ matrix.pyver }} 
 
+      - name: Download wheels artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: wheels
+          path: dist
+
       - name: Build windows app archive
         run: etc/release/scancode-create-release-app-windows.sh ${{ matrix.pyver }}
 
@@ -195,9 +191,13 @@ jobs:
     name: Build app source
     runs-on: ubuntu-22.04
 
+    needs:
+      - build_scancode_wheels_for_pypi
+
     defaults:
       run:
         shell: bash
+
     strategy:
       fail-fast: true
 
@@ -209,6 +209,12 @@ jobs:
         with:
           python-version: "3.10"
 
+      - name: Download wheels artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: wheels
+          path: dist
+
       - name: Build source archive with deps
         run: etc/release/scancode-create-release-app-sources.sh
 
@@ -235,7 +241,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        os: [ubuntu-22.04, ubuntu-22.04, macos-11, macos-12]
+        os: [ubuntu-20.04, ubuntu-22.04, macos-11, macos-12]
         pyver: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
@@ -249,7 +255,7 @@ jobs:
       - name: Download wheels artifact
         uses: actions/download-artifact@v3
         with:
-          name: wheels-${{ matrix.pyver }}
+          name: wheels
           path: dist
 
       - name: test install wheels
@@ -258,7 +264,7 @@ jobs:
           rm -rf venv;  \
           python -m venv venv
           venv/bin/python -m pip install --upgrade pip wheel setuptools
-          for f in `find dist -type f -name "*${{ matrix.pyver }}*"`; \
+          for f in `find dist -type f -name "scancode*${{ matrix.pyver }}*"`; \
             do \
               venv/bin/python -m pip install --force-reinstall "$f[full]" ; \
               venv/bin/scancode -clipeu --json-pp - some.file ; \
@@ -295,7 +301,7 @@ jobs:
       - name: Download wheels artifact
         uses: actions/download-artifact@v3
         with:
-          name: wheels-${{ matrix.pyver }}
+          name: wheels
           path: dist
 
       - name: test install wheel

CHANGELOG.rst

@@ -4,6 +4,12 @@ Changelog
 v33.0.0 (next next, roadmap)
 ----------------------------
 
+- We changed the way licenses text are packaged to help creating smaller packages for publishing
+  on PyPI. If you relied on actual license text files present in the wheels, you will now need
+  to install the package scancode-toolkit[licenses] that will also contain the files. However, the
+  better and preferred way is to run the command line "scancode-license-data" that will create
+  a directory with all license files, and this works with all builds of scancode-toolkit.
+
 - We now support new package manifest formats:
 
   - OpenWRT packages.
@@ -34,7 +40,11 @@ v33.0.0 (next next, roadmap)
   of these in other summary plugins.
   See https://github.com/nexB/scancode-toolkit/issues/1745
 
-v32.1.0 (next, roadmap)
+- The Python API of the packagedcode module has changed. The package datafile "handlers" registry
+  that used to be in packagedcode/__init__.py has been moved to packagedcode/handlers.py
+
+
+v32.1.0 2024-03-22
 ----------------------------
 
 New CLI options:

configure

@@ -149,11 +149,12 @@ if [[ $OSTYPE == 'darwin'* ]]; then
 fi
 
 # Requirement arguments passed to pip and used by default or with --dev.
-REQUIREMENTS="--editable $BASE --constraint requirements.txt --constraint requirements-linux.txt"
-DEV_REQUIREMENTS="--editable $BASE_DEV --constraint requirements.txt --constraint requirements-linux.txt --constraint requirements-dev.txt"
-DOCS_REQUIREMENTS="--editable .[docs] --constraint requirements.txt"
-REL_REQUIREMENTS="--requirement etc/scripts/requirements.txt"
-PROD_REQUIREMENTS="scancode_toolkit*.whl"
+REQUIREMENTS="--no-build-isolation --editable $BASE --constraint requirements.txt --constraint requirements-linux.txt"
+DEV_REQUIREMENTS="--no-build-isolation --editable $BASE_DEV --constraint requirements.txt --constraint requirements-linux.txt --constraint requirements-dev.txt"
+DOCS_REQUIREMENTS="--no-build-isolation --editable .[docs] --constraint requirements.txt"
+REL_REQUIREMENTS="--no-build-isolation --requirement etc/scripts/requirements.txt"
+PROD_REQUIREMENTS="--no-build-isolation scancode_toolkit*.whl"
+FLOT_REQUIREMENTS="etc/thirdparty/flot*.whl"
 
 # where we create a virtualenv
 VIRTUALENV_DIR=venv
@@ -175,16 +176,10 @@ CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin
 
 
-################################
-# Install with or without and index. With "--no-index" this is using only local wheels
-# This is an offline mode with no index and no network operations
-# NO_INDEX="--no-index "
-NO_INDEX=""
-
-
 ################################
 # Thirdparty package locations and index handling
 # Find packages from the local thirdparty directory if present
+# offline mode for scancode installation with no Pypi index at all
 thirddir=$CFG_ROOT_DIR/thirdparty
 if [[ "$(echo $thirddir/*.whl)x" != "$thirddir/*.whlx" ]]; then
     PIP_EXTRA_ARGS="--no-index --find-links $CFG_ROOT_DIR/thirdparty"
@@ -257,7 +252,6 @@ install_packages() {
 
     "$CFG_BIN_DIR/pip" install \
         --upgrade \
-        --no-build-isolation \
         $CFG_QUIET \
         $PIP_EXTRA_ARGS \
         $1
@@ -320,6 +314,7 @@ PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS"
 
 find_python
 create_virtualenv "$VIRTUALENV_DIR"
+install_packages "$FLOT_REQUIREMENTS"
 install_packages "$CFG_REQUIREMENTS"
 . "$CFG_BIN_DIR/activate"
 

configure.bat

@@ -26,10 +26,13 @@
 @rem ################################
 
 @rem # Requirement arguments passed to pip and used by default or with --dev.
-set "REQUIREMENTS=--editable . --constraint requirements.txt"
-set "DEV_REQUIREMENTS=--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt"
-set "DOCS_REQUIREMENTS=--editable .[docs] --constraint requirements.txt"
-set "REL_REQUIREMENTS=--requirement etc/scripts/requirements.txt"
+set "REQUIREMENTS=--no-build-isolation --editable . --constraint requirements.txt"
+set "DEV_REQUIREMENTS=--no-build-isolation --editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt"
+set "DOCS_REQUIREMENTS=--no-build-isolation --editable .[docs] --constraint requirements.txt"
+set "REL_REQUIREMENTS=--no-build-isolation --requirement etc/scripts/requirements.txt"
+set "PROD_REQUIREMENTS=--no-build-isolation scancode_toolkit*.whl"
+set "FLOT_REQUIREMENTS=etc/thirdparty/flot*.whl"
+
 
 @rem # where we create a virtualenv
 set "VIRTUALENV_DIR=venv"
@@ -54,7 +57,7 @@ set "CFG_BIN_DIR=%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts"
 @rem ################################
 @rem # Thirdparty package locations and index handling
 @rem # Find packages from the local thirdparty directory or from thirdparty.aboutcode.org
-@rem # offline mode for scancode installation with no index at all
+@rem # offline mode for scancode installation with no Pypi index at all
 if exist "%CFG_ROOT_DIR%\thirdparty" (
     set PIP_EXTRA_ARGS=--no-index --find-links "%CFG_ROOT_DIR%\thirdparty"
     for %%i in (scancode_toolkit*.whl) do set "CFG_REQUIREMENTS=%CFG_ROOT_DIR%\%%i[full]"
@@ -164,7 +167,12 @@ if %ERRORLEVEL% neq 0 (
 
 "%CFG_BIN_DIR%\pip" install ^
     --upgrade ^
-    --no-build-isolation ^
+    %CFG_QUIET% ^
+    %PIP_EXTRA_ARGS% ^
+    %FLOT_REQUIREMENTS%
+
+"%CFG_BIN_DIR%\pip" install ^
+    --upgrade ^
     %CFG_QUIET% ^
     %PIP_EXTRA_ARGS% ^
     %CFG_REQUIREMENTS%