talipot-python: Create a virtualenv for the embedded interpreter (WIP)

.github/workflows/code-coverage.yml

@@ -37,9 +37,7 @@ jobs:
           graphviz
           xvfb
       - name: Install sip
-        run: |
-          sudo pip3 install --upgrade pip
-          sudo pip3 install sip
+        run: sudo pip3 install sip
       - name: Prepare ccache timestamp
         id: get-current-date
         run: |
@@ -66,7 +64,7 @@ jobs:
           -DTALIPOT_CODE_COVERAGE=ON
       - name: Talipot build
         working-directory: ./build
-        run: ninja -j4
+        run: ninja -j4 install
       - name: Run Talipot unit tests
         working-directory: ./build
         run: xvfb-run ninja tests

.github/workflows/macos-homebrew-build.yml

@@ -26,6 +26,7 @@ jobs:
         run: brew install
           ccache
           cmake
+          coreutils
           llvm
           qhull
           yajl
@@ -114,9 +115,11 @@ jobs:
           hdiutil attach Talipot*.dmg
           cp -r /Volumes/Talipot*/Talipot*.app /Applications/
           sudo xattr -r -d com.apple.quarantine /Applications/Talipot*.app
+          rm -rf ~/.Talipot*
           /Applications/Talipot*.app/Contents/MacOS/Talipot \
             --check-application-starts \
             --debug-plugins-load
+          /Users/runner/.Talipot-1.0/venv3.13/bin/python3 -m ensurepip --upgrade --default-pip
           hdiutil detach /Volumes/Talipot*
       - name: Upload Talipot bundle to GitHub Actions artifacts
         uses: actions/upload-artifact@v4

.github/workflows/macos-macports-build.yml

@@ -50,6 +50,7 @@ jobs:
           cmake
           clang-${{ env.CLANG_VERSION }}
           ccache
+          coreutils
           zlib
           qhull
           yajl
@@ -125,9 +126,11 @@ jobs:
           hdiutil attach Talipot*.dmg
           cp -r /Volumes/Talipot*/Talipot*.app /Applications/
           sudo xattr -r -d com.apple.quarantine /Applications/Talipot*.app
+          rm -rf ~/.Talipot*
           /Applications/Talipot*.app/Contents/MacOS/Talipot \
             --check-application-starts \
             --debug-plugins-load
+          /Users/runner/.Talipot-1.0/venv3.13/bin/python3 -m ensurepip --upgrade --default-pip
           hdiutil detach /Volumes/Talipot*
       - name: Upload Talipot bundle to GitHub Actions artifacts
         uses: actions/upload-artifact@v4

.github/workflows/windows-mingw64-build.yml

@@ -7,6 +7,8 @@ jobs:
   mingw64:
     name: ${{ matrix.config.name }}
     runs-on: windows-latest
+    env:
+      PYTHON_VERSION: "3.13"
     defaults:
       run:
         shell: msys2 {0}
@@ -40,16 +42,23 @@ jobs:
             mingw-w64-${{ matrix.config.arch }}-qhull
             mingw-w64-${{ matrix.config.arch }}-graphviz
             mingw-w64-${{ matrix.config.arch }}-libgit2
-            mingw-w64-${{ matrix.config.arch }}-python
             mingw-w64-${{ matrix.config.arch }}-cppunit
             mingw-w64-${{ matrix.config.arch }}-fontconfig
             mingw-w64-${{ matrix.config.arch }}-freetype
             mingw-w64-${{ matrix.config.arch }}-fribidi
             mingw-w64-${{ matrix.config.arch }}-glew
             mingw-w64-${{ matrix.config.arch }}-qt5
             mingw-w64-${{ matrix.config.arch }}-quazip
-            mingw-w64-${{ matrix.config.arch }}-python-sphinx
-            mingw-w64-${{ matrix.config.arch }}-sip
+      - name: Install Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@v5
+        id: python-install
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+      - name: Install sip and sphinx
+        run: |
+          set PATH=%Python3_ROOT_DIR%\Scripts:%PATH%
+          pip install sip sphinx
+        shell: cmd
       - name: Prepare ccache timestamp
         id: get-current-date
         run: |
@@ -75,7 +84,7 @@ jobs:
           -DCMAKE_BUILD_TYPE=Release
           -DCMAKE_NEED_RESPONSE=ON
           -DCMAKE_INSTALL_PREFIX=$PWD/install
-          -DPython3_EXECUTABLE=/${{ matrix.config.msystem }}/bin/python3
+          -DPython3_EXECUTABLE=$Python3_ROOT_DIR/python.exe
           -DTALIPOT_BUILD_TESTS=ON
           -DTALIPOT_USE_CCACHE=ON ..
       - name: Talipot build

bundlers/linux/make_appimage_bundle.sh.in

@@ -103,6 +103,7 @@ export LD_LIBRARY_PATH=${QT_INSTALL_LIBS_DIR}:${LD_LIBRARY_PATH}
 export LD_LIBRARY_PATH=$(dirname @PYTHON_LIBRARY@):$LD_LIBRARY_PATH
 
 # add Python environment
+cp -v -p $TALIPOT_INSTALL_DIR/bin/python3* $BUNDLE_BIN_DIR
 rm -f $BUNDLE_LIB_DIR/libpython*
 PYTHON_LIB=$(ldd $(ls $BUNDLE_LIB_DIR/libtalipot-python*) | \
   grep libpython | awk '{print $3}')
@@ -125,9 +126,16 @@ if [ "$PYTHON_LIB" != "" ]; then
   fi
   echo "copying $(dirname $PYTHON_LIB)/$PYTHON_PACKAGE_DIR files into \
   $PYTHON_PACKAGE_BUNDLE_DIR"
-  find . \( -type f \) \( ! -name "*.pyc" \) \( ! -name "*.pyo" \) -exec \
+  find . \( -type f \) \( ! -name "*.pyo" \) -exec \
     cp --parents --preserve=mode {} $PYTHON_PACKAGE_BUNDLE_DIR \;
   popd > /dev/null 2>&1
+  mkdir -p $PYTHON_PACKAGE_BUNDLE_DIR/ensurepip/_bundled
+  pushd $PYTHON_PACKAGE_BUNDLE_DIR/ensurepip/_bundled > /dev/null 2>&1
+  setuptools_version=$(python3.11 -c "import ensurepip;print(ensurepip._SETUPTOOLS_VERSION)")
+  pip_version=$(python3.11 -c "import ensurepip;print(ensurepip._PIP_VERSION)")
+  wget https://pypi.debian.net/setuptools/setuptools-${setuptools_version}-py3-none-any.whl
+  wget https://pypi.debian.net/pip/pip-${pip_version}-py3-none-any.whl
+  popd > /dev/null 2>&1
 fi
 
 # copy required shared libs using linuxdeployqt tool

bundlers/mac/CMakeLists.txt

@@ -4,5 +4,5 @@ CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/mac_bundle.sh.in"
 ADD_CUSTOM_TARGET(
   bundle
   COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_install.cmake
-  COMMAND sh mac_bundle.sh ${CMAKE_BINARY_DIR}
+  COMMAND bash mac_bundle.sh ${CMAKE_BINARY_DIR}
   WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})

bundlers/mac/mac_bundle.sh.in

@@ -67,27 +67,32 @@ cp -r $GV_DIR Frameworks/graphviz
 echo 'Copying Python Framework'
 mkdir -p Frameworks/Python.framework/Versions/@PYTHON_VERSION@
 cp -r @PYTHON_STDLIB_DIR@/../../* Frameworks/Python.framework/Versions/@PYTHON_VERSION@/
-find Frameworks/Python.framework/ | grep -E "(__pycache__|\.pyc|\.pyo$)" | xargs rm -rf
+find Frameworks/Python.framework/ | grep -E "(__pycache__|\.pyo$)" | xargs rm -rf
 rm -rf Frameworks/Python.framework/Versions/@PYTHON_VERSION@/share
+rm -f bin/python3*
+ln -s ../Frameworks/Python.framework/Versions/@PYTHON_VERSION@/bin/python3 bin/python3
 
 echo 'Copying License'
 cd "${DEST_DIR}/application"
 cp "${SRC_DIR}/../../LICENSE" .
 
 QT_LIB_DIR="@QT_QTCORE_LIBRARY@/.."
 TALIPOT_APP=${DEST_DIR}/application/Talipot.app
-echo 'Copying Resources'
+
+echo 'Copying Qt Resources'
 cd "${TALIPOT_APP}/Contents/Frameworks/"
 cp -r "${QT_LIB_DIR}/QtGui.framework/Resources/qt_menu.nib" ../Resources 2>/dev/null
 cp "@QT_QTCLUCENE_LIBRARY@" . 2>/dev/null
 
+echo 'Executing macdeployqt'
 cd ..
 # configure talipot
 mv bin/talipot MacOS/Talipot
 # tell macdeployqt to also process Talipot Python dynamic module (not processed by default as
 # its extension is .so instead of .dylib)
 talipot_python_module=$(ls ${TALIPOT_APP}/Contents/lib/talipot/python/talipot/native/talipot*.so)
 mac_deploy_qt_opts=-executable=${talipot_python_module}
+
 if [ $(echo ${QT_VERSION} | cut -c1) -ge 6 ]
 then
     # qt plugins end up with broken rpaths when using macdeployt from Qt6
@@ -100,7 +105,7 @@ ${QT_BINARY_DIR}/macdeployqt ${TALIPOT_APP} $mac_deploy_qt_opts
 
 # ensure clang libc++* are present in bundle
 LIB_CXX_DIR=$(echo "@CMAKE_SHARED_LINKER_FLAGS@" | cut -f1 -d" " | cut -c3-)
-if [ -d ${LIB_CXX_DIR} ]
+if [ -f ${LIB_CXX_DIR}/libc++.1.dylib ]
 then
     cp ${LIB_CXX_DIR}/libc++.1.dylib ${TALIPOT_APP}/Contents/Frameworks/
     cp ${LIB_CXX_DIR}/libc++abi.1.dylib ${TALIPOT_APP}/Contents/Frameworks/
@@ -127,6 +132,38 @@ platforms position styles tls
     done
 fi
 
+echo 'Fix remaining hardcoded dylib loading paths'
+# fix remaining hardcoded dylib loading paths in binaries to ensure bundle portability
+pushd "${DEST_DIR}/application/Talipot.app/Contents/" > /dev/null 2>&1
+export PATH=/opt/local/libexec/gnubin:$PATH
+realpath_cmd=$(which grealpath || which realpath)
+for binary in $(find . -perm +0111 -type f)
+do
+    for dylib in $(otool -L $binary | \
+        grep -E '/opt/local/|/usr/local/|/opt/homebrew/' | cut -d '(' -s -f 1 | xargs)
+    do
+        for pattern in /Frameworks/ /lib/
+        do
+            before_pattern=${dylib%%"$pattern"*}
+            if [ "$before_pattern" != "$dylib" ]
+            then
+                let pos=${#before_pattern}+${#pattern}
+                dylib_subpath=${dylib:${pos}}
+
+                binary_dir=$(dirname $binary)
+                rel_path=$($realpath_cmd  --relative-to=$binary_dir ./Frameworks/$dylib_subpath)
+                loader_path="@loader_path/$rel_path"
+
+                echo "install_name_tool -change $dylib $loader_path $binary"
+                install_name_tool -change $dylib $loader_path $binary
+
+                break
+            fi
+        done
+    done
+done
+popd > /dev/null 2>&1
+
 mv MacOS/Talipot bin/talipot
 # rename
 mv ${TALIPOT_APP} ${DEST_DIR}/application/${APP_NAME}-@[email protected]

library/talipot-python/include/talipot/PythonInterpreter.h

@@ -1,6 +1,6 @@
 /**
  *
- * Copyright (C) 2019-2021  The Talipot developers
+ * Copyright (C) 2019-2024  The Talipot developers
  *
  * Talipot is a fork of Tulip, created by David Auber
  * and the Tulip development Team from LaBRI, University of Bordeaux
@@ -51,6 +51,7 @@ class TLP_PYTHON_SCOPE PythonInterpreter : public QObject, public Singleton<Pyth
 
   void setDefaultConsoleWidget(QAbstractScrollArea *consoleWidget);
   void setConsoleWidget(QAbstractScrollArea *consoleWidget);
+  void setupVirtualEnv();
 
   bool _wasInit;
   bool _runningScript;

library/talipot-python/src/PythonInterpreter.cpp

@@ -1,6 +1,6 @@
 /**
  *
- * Copyright (C) 2019-2024  The Talipot developers
+ * Copyright (C) 2019-2025  The Talipot developers
  *
  * Talipot is a fork of Tulip, created by David Auber
  * and the Tulip development Team from LaBRI, University of Bordeaux
@@ -27,6 +27,7 @@
 #if defined(__MINGW32__)
 #include <QSslSocket>
 #endif
+#include <QFileInfo>
 
 #include <talipot/Release.h>
 #include <talipot/PythonVersionChecker.h>
@@ -133,8 +134,10 @@ int tracefunc(PyObject *, PyFrameObject *, int what, PyObject *) {
 const QString PythonInterpreter::pythonPluginsPath(tlpStringToQString(tlp::TalipotLibDir) +
                                                    "talipot/python/");
 
-const QString PythonInterpreter::pythonPluginsPathHome(QDir::homePath() + "/.Talipot-" +
-                                                       TALIPOT_MM_VERSION + "/plugins/python");
+const QString talipotUserDirectory = QDir::homePath() + "/.Talipot-" + TALIPOT_MM_VERSION;
+const QString talipotVenvDirectory =
+    talipotUserDirectory + "/venv" + PythonVersionChecker::compiledVersion();
+const QString PythonInterpreter::pythonPluginsPathHome(talipotUserDirectory + "/plugins/python");
 
 const char PythonInterpreter::pythonReservedCharacters[] = {
     '#', '%',  '/',  '+', '-', '&', '*', '<', '>', '|', '~', '^', '=',
@@ -211,6 +214,10 @@ PythonInterpreter::PythonInterpreter()
 #endif
   }
 
+#ifndef MSYS2_PYTHON
+  setupVirtualEnv();
+#endif
+
   holdGIL();
 
   importModule("sys");
@@ -305,6 +312,61 @@ PythonInterpreter::PythonInterpreter()
   }
 
   releaseGIL();
+
+#ifndef MSYS2_PYTHON
+  setupVirtualEnv();
+#endif
+}
+
+void PythonInterpreter::setupVirtualEnv() {
+#ifdef Q_OS_WIN
+  if (!QFileInfo(talipotVenvDirectory + "/Scripts/pip.exe").exists()) {
+#else
+  if (!QFileInfo(talipotVenvDirectory + "/bin/pip").exists()) {
+#endif
+    runString(QString(R"(
+import os
+import platform
+import sys
+import venv
+python_command = 'python3'
+if platform.system() == 'Windows':
+    python_command = 'python.exe'
+sys._base_executable = os.path.join('%1', python_command)
+os.environ['LD_LIBRARY_PATH'] = '%2';
+os.environ['DYLD_FALLBACK_LIBRARY_PATH'] = '%2';
+os.environ['DYLD_FRAMEWORK_PATH'] = '%2';
+venv.create('%3', with_pip=False, symlinks=True)
+)")
+                  .arg(QApplication::applicationDirPath(), tlpStringToQString(tlp::TalipotLibDir),
+                       talipotVenvDirectory));
+  }
+
+  runString(QString(R"(
+import os
+import platform
+import sys
+
+base = '%1'
+if platform.system() == 'Windows':
+    site_packages = os.path.join(base, 'Lib', 'site-packages')
+else:
+    site_packages = os.path.join(
+        base, 'lib',
+        'python%s.%s' % (sys.version_info.major, sys.version_info.minor),
+        'site-packages')
+prev_sys_path = list(sys.path)
+import site
+site.addsitedir(site_packages)
+sys.real_prefix = sys.prefix
+sys.prefix = base
+new_sys_path = []
+for item in list(sys.path):
+    if item not in prev_sys_path:
+        new_sys_path.append(item)
+        sys.path.remove(item)
+sys.path[:0] = new_sys_path)")
+                .arg(talipotVenvDirectory));
 }
 
 PythonInterpreter::~PythonInterpreter() {

software/talipot/CMakeLists.txt

@@ -269,3 +269,23 @@ IF(LINUX)
           OUTPUT_QUIET ERROR_QUIET)")
   ENDIF(TALIPOT_LINUX_DESKTOP_REGISTRATION)
 ENDIF(LINUX)
+
+IF(NOT MSYS2_PYTHON)
+  STRING(REPLACE "\\" "/" PYTHON_EXE_PATH "${PYTHON_EXECUTABLE}")
+  GET_FILENAME_COMPONENT(PYTHON_EXE_NAME "${PYTHON_EXECUTABLE}" NAME)
+
+  INSTALL(
+    CODE "
+FILE(COPY \"${PYTHON_EXE_PATH}\"
+     DESTINATION \"\${CMAKE_INSTALL_PREFIX}/bin/\" FOLLOW_SYMLINK_CHAIN)
+IF(NOT WIN32
+  AND NOT ${PYTHON_EXE_NAME} STREQUAL python3
+  AND NOT EXISTS ${CMAKE_INSTALL_PREFIX}/bin/python3)
+  FILE(CREATE_LINK ${CMAKE_INSTALL_PREFIX}/bin/${PYTHON_EXE_NAME}
+       ${CMAKE_INSTALL_PREFIX}/bin/python3 SYMBOLIC)
+ENDIF(
+  NOT WIN32
+  AND NOT ${PYTHON_EXE_NAME} STREQUAL python3
+  AND NOT EXISTS ${CMAKE_INSTALL_PREFIX}/bin/python3)
+")
+ENDIF(NOT MSYS2_PYTHON)