Publish licensedcode data/index as seperate pypi wheels

.github/workflows/licensedcode-data-index-release.yml

@@ -0,0 +1,90 @@
+name: Create library release archives, create a GH release and publish PyPI wheel and sdist on tag in main branch
+
+
+# This is executed automatically on a tag in the main branch
+
+# Summary of the steps:
+# - build wheels and sdist
+# - upload wheels and sdist to PyPI
+# - create gh-release and upload wheels and dists there
+# TODO: smoke test wheels and sdist
+# TODO: add changelog to release text body
+
+# WARNING: this is designed only for packages building as pure Python wheels
+
+on:
+  workflow_dispatch:
+  push:
+    tags:
+      - "licensedcode-v*.*.*"
+
+jobs:
+  build-pypi-distribs:
+    permissions:
+      contents: read # to fetch code (actions/checkout)
+
+    name: Build and publish library to PyPI
+    runs-on: ubuntu-24.04
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.14
+
+      - name: Install pypa/build and twine
+        run: python -m pip install --user --upgrade build twine pkginfo flot
+
+      - name: Install requirements then build main and mini wheel
+        run: etc/release/licensedcode-create-pypi-wheel-and-sdist.sh
+
+      - name: Upload built archives
+        uses: actions/upload-artifact@v4
+        with:
+          name: pypi_archives
+          path: dist/*
+
+
+  create-gh-release:
+    permissions:
+      contents: write
+
+    name: Create GH release
+    needs:
+      - build-pypi-distribs
+    runs-on: ubuntu-24.04
+
+    steps:
+      - name: Download built archives
+        uses: actions/download-artifact@v4
+        with:
+          name: pypi_archives
+          path: dist
+
+      - name: Create GH release
+        uses: softprops/action-gh-release@v2
+        with:
+          draft: true
+          files: dist/*
+
+
+  create-pypi-release:
+    name: Create PyPI release
+    needs:
+      - create-gh-release
+    runs-on: ubuntu-24.04
+    environment: pypi-publish
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Download built archives
+        uses: actions/download-artifact@v4
+        with:
+          name: pypi_archives
+          path: dist
+
+      - name: Publish to PyPI
+        if: startsWith(github.ref, 'refs/tags')
+        uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/scancode-release.yml

@@ -20,7 +20,7 @@ on:
 permissions: {}
 jobs:
 
-  build_scancode_wheel_for_pypi:
+  build_scancode_wheels_sdist_for_pypi:
     permissions:
       contents: read # to fetch code (actions/checkout)
 
@@ -31,59 +31,25 @@ jobs:
       run:
         shell: bash
 
-    strategy:
-      fail-fast: true
-      matrix:
-       pyver: ["3.10", "3.11", "3.12", "3.13", "3.14"]
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Python on ${{ matrix.pyver }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.pyver }} 
-
-      - name: Install requirements then build main and mini wheel
-        run: etc/release/scancode-create-pypi-wheel.sh
-
-      - name: Collect built wheels for ${{ matrix.pyver }} 
-        uses: actions/upload-artifact@v4
-        with:
-          name: wheels-${{ matrix.pyver }}
-          path: dist/*.whl
-
-
-  build_scancode_sdist_for_pypi:
-    permissions:
-      contents: read # to fetch code (actions/checkout)
-
-    name: Build PyPI sdist archives
-    runs-on: ubuntu-24.04
-
-    defaults:
-      run:
-        shell: bash
-
-    strategy:
-      fail-fast: true
-
     steps:
       - uses: actions/checkout@v4
 
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.14"
+          python-version: 3.14
+
+      - name: Install requirements then build main and mini wheel, sdist
+        run: etc/release/scancode-create-pypi-wheel-and-sdist.sh
 
-      - name: Install requirements then build main and mini sdist
-        run: etc/release/scancode-create-pypi-sdist.sh
+      - name: Install requirements then build licensedcode data/index wheels, sdist
+        run: etc/release/licensedcode-create-pypi-wheel-and-sdist.sh
 
-      - name: Collect built sdist
+      - name: Collect built wheels and sdist
         uses: actions/upload-artifact@v4
         with:
-          name: sdists
-          path: dist/*.tar.gz
+          name: wheels_sdist
+          path: dist/*
 
 
   build_scancode_for_release_linux:
@@ -229,7 +195,7 @@ jobs:
 
     name: Test PyPI wheels on linux and mac
     needs:
-      - build_scancode_wheel_for_pypi
+      - build_scancode_wheels_sdist_for_pypi
     runs-on: ${{ matrix.os }}
 
     defaults:
@@ -253,16 +219,20 @@ jobs:
       - name: Download wheels artifact
         uses: actions/download-artifact@v4
         with:
-          name: wheels-${{ matrix.pyver }}
+          name: wheels_sdist
           path: dist
 
-      - name: test install wheels
+      - name: install and test wheels
         run: |
           echo "license: gpl-2.0" > some.file
           rm -rf venv;  \
           python -m venv venv
           venv/bin/python -m pip install --upgrade pip wheel setuptools
-          for f in `find dist -type f -name "*${{ matrix.pyver }}*"`; \
+          for f in `find dist -type f -name "licensedcode*.whl"`; \
+            do \
+              venv/bin/python -m pip install --force-reinstall "$f" ; \
+            done
+          for f in `find dist -type f -name "scancode_toolkit-*.whl"`; \
             do \
               venv/bin/python -m pip install --force-reinstall "$f[full]" ; \
               venv/bin/scancode -clipeu --json-pp - some.file ; \
@@ -275,7 +245,7 @@ jobs:
 
     name: Test PyPI wheels on Windows
     needs:
-      - build_scancode_wheel_for_pypi
+      - build_scancode_wheels_sdist_for_pypi
     runs-on: ${{ matrix.os }}
 
     defaults:
@@ -299,15 +269,19 @@ jobs:
       - name: Download wheels artifact
         uses: actions/download-artifact@v4
         with:
-          name: wheels-${{ matrix.pyver }}
+          name: wheels_sdist
           path: dist
 
-      - name: test install wheel
+      - name: install and test wheels
         run: |
           echo "license: gpl-2.0" > some.file
           python -m venv venv
           venv/Scripts/python.exe -m pip install --upgrade pip
-          for f in `find dist -type f  -name "*${{ matrix.pyver }}*"`; \
+          for f in `find dist -type f -name "licensedcode*.whl"`; \
+            do \
+              venv/Scripts/python.exe -m pip install --force-reinstall "$f" ; \
+            done
+          for f in `find dist -type f  -name "scancode_toolkit-*.whl"`; \
             do \
               venv/Scripts/python.exe -m pip install --force-reinstall "$f[full]" ; \
               venv/Scripts/scancode -clipeu --json-pp - some.file ; \
@@ -454,6 +428,7 @@ jobs:
       - build_scancode_for_release_windows
       - build_scancode_for_release_linux
       - build_scancode_for_release_macos
+      - build_scancode_wheels_sdist_for_pypi
 
     runs-on: ubuntu-24.04
     defaults:
@@ -559,6 +534,12 @@ jobs:
           name: windows_app_py_3.14
           path: dist
 
+      - name: Download scancode and licensedcode wheels
+        uses: actions/download-artifact@v4
+        with:
+          name: wheels_sdist
+          path: dist
+
       - name: Mock GH release
         run: |
           ls -al dist
@@ -573,8 +554,7 @@ jobs:
   publish_to_pypi:
     name: Publish to PyPI
     needs:
-      - build_scancode_wheel_for_pypi
-      - build_scancode_sdist_for_pypi
+      - build_scancode_wheels_sdist_for_pypi
       - smoke_test_install_and_run_pypi_wheels_on_windows
       - smoke_test_install_and_run_pypi_wheels_on_posix
       - publish_to_gh_release
@@ -585,26 +565,23 @@ jobs:
     defaults:
       run:
         shell: bash
-    strategy:
-      fail-fast: false
-      matrix:
-        dist_names: ["wheels-3.10", "wheels-3.11", "wheels-3.12", "wheels-3.13", "wheels-3.14", "sdists"]
 
     steps:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
           python-version: 3.14
 
-      - name: Download a single artifact
+      - name: scancode and licensedcode wheels
         uses: actions/download-artifact@v4
         with:
-          name: ${{ matrix.dist_names }}
+          name: wheels_sdist
           path: dist
 
       - name: Mock PyPI upload
         run: |
           ls -al dist
 
       - name: Publish distributions to PyPI
+        if: startsWith(github.ref, 'refs/tags')
         uses: pypa/gh-action-pypi-publish@release/v1

configure

@@ -142,16 +142,17 @@ CLI_ARGS=$1
 ################################
 
 BASE=".[packages]"
-BASE_DEV=".[packages,testing]"
+BASE_DEV=".[packages,dev]"
 if [[ $OSTYPE == 'darwin'* ]]; then
     BASE="."
-    BASE_DEV=".[testing]"
+    BASE_DEV=".[dev]"
 fi
 
 # Requirement arguments passed to pip and used by default or with --dev.
-REQUIREMENTS="--editable . --constraint requirements.txt"
-DEV_REQUIREMENTS="--editable .[dev,packages] --constraint requirements.txt --constraint requirements-dev.txt"
-PROD_REQUIREMENTS="scancode_toolkit*.whl"
+REQUIREMENTS="--no-build-isolation --editable $BASE --constraint requirements.txt --constraint requirements-linux.txt"
+DEV_REQUIREMENTS="--no-build-isolation --editable $BASE_DEV --constraint requirements.txt --constraint requirements-linux.txt --constraint requirements-dev.txt"
+PROD_REQUIREMENTS="--no-build-isolation scancode_toolkit*.whl licensedcode_index*.whl"
+FLOT_REQUIREMENTS="etc/thirdparty/flot*.whl"
 
 # where we create a virtualenv
 VIRTUALENV_DIR=venv
@@ -173,16 +174,10 @@ CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin
 
 
-################################
-# Install with or without and index. With "--no-index" this is using only local wheels
-# This is an offline mode with no index and no network operations
-# NO_INDEX="--no-index "
-NO_INDEX=""
-
-
 ################################
 # Thirdparty package locations and index handling
 # Find packages from the local thirdparty directory if present
+# offline mode for scancode installation with no Pypi index at all
 thirddir=$CFG_ROOT_DIR/thirdparty
 if [[ "$(echo $thirddir/*.whl)x" != "$thirddir/*.whlx" ]]; then
     PIP_EXTRA_ARGS="--no-index --find-links $CFG_ROOT_DIR/thirdparty"
@@ -255,7 +250,6 @@ install_packages() {
     # We always have the PEP517 build dependencies installed already.
     "$CFG_BIN_DIR/pip" install \
         --upgrade \
-        --no-build-isolation \
         $CFG_QUIET \
         $PIP_EXTRA_ARGS \
         $1
@@ -317,6 +311,7 @@ PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS"
 
 find_python
 create_virtualenv "$VIRTUALENV_DIR"
+install_packages "$FLOT_REQUIREMENTS"
 install_packages "$CFG_REQUIREMENTS"
 . "$CFG_BIN_DIR/activate"
 "$CFG_BIN_DIR/scancode-train-gibberish-model"

configure.bat

@@ -26,9 +26,10 @@
 @rem ################################
 
 @rem # Requirement arguments passed to pip and used by default or with --dev.
-set "REQUIREMENTS=--editable . --constraint requirements.txt"
-set "DEV_REQUIREMENTS=--editable .[dev] --constraint requirements.txt --constraint requirements-dev.txt"
-set "PROD_REQUIREMENTS=scancode_toolkit*.whl"
+set "REQUIREMENTS=--no-build-isolation --editable . --constraint requirements.txt"
+set "DEV_REQUIREMENTS=--no-build-isolation --editable .[dev] --constraint requirements.txt --constraint requirements-dev.txt"
+set "PROD_REQUIREMENTS=--no-build-isolation scancode_toolkit*.whl licensedcode_index*.whl"
+set "FLOT_REQUIREMENTS=etc/thirdparty/flot*.whl"
 
 @rem # where we create a virtualenv
 set "VIRTUALENV_DIR=venv"
@@ -157,10 +158,16 @@ if %ERRORLEVEL% neq 0 (
 
 "%CFG_BIN_DIR%\pip" install ^
     --upgrade ^
-    --no-build-isolation ^
+    %CFG_QUIET% ^
+    %PIP_EXTRA_ARGS% ^
+    %FLOT_REQUIREMENTS%
+
+"%CFG_BIN_DIR%\pip" install ^
+    --upgrade ^
     %CFG_QUIET% ^
     %PIP_EXTRA_ARGS% ^
     %CFG_REQUIREMENTS%
+
 "%CFG_BIN_DIR%\scancode-train-gibberish-model"
 
 @rem ################################