From a43043931905ee4bf74a655b20051e69cf4ea0cb Mon Sep 17 00:00:00 2001 From: Antoine Lambert Date: Tue, 31 Dec 2024 11:44:49 +0100 Subject: [PATCH] talipot-python: Create a virtualenv for the embedded interpreter (WIP) --- .github/workflows/code-coverage.yml | 6 +- .github/workflows/macos-homebrew-build.yml | 3 + .github/workflows/macos-macports-build.yml | 3 + .github/workflows/windows-mingw64-build.yml | 17 +++-- bundlers/linux/make_appimage_bundle.sh.in | 10 ++- bundlers/mac/CMakeLists.txt | 2 +- bundlers/mac/mac_bundle.sh.in | 43 +++++++++++- .../include/talipot/PythonInterpreter.h | 3 +- .../talipot-python/src/PythonInterpreter.cpp | 68 ++++++++++++++++++- software/talipot/CMakeLists.txt | 20 ++++++ 10 files changed, 158 insertions(+), 17 deletions(-) diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index 92a36ca69c..5294de3127 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -37,9 +37,7 @@ jobs: graphviz xvfb - name: Install sip - run: | - sudo pip3 install --upgrade pip - sudo pip3 install sip + run: sudo pip3 install sip - name: Prepare ccache timestamp id: get-current-date run: | @@ -66,7 +64,7 @@ jobs: -DTALIPOT_CODE_COVERAGE=ON - name: Talipot build working-directory: ./build - run: ninja -j4 + run: ninja -j4 install - name: Run Talipot unit tests working-directory: ./build run: xvfb-run ninja tests diff --git a/.github/workflows/macos-homebrew-build.yml b/.github/workflows/macos-homebrew-build.yml index 499c6f563c..c2d8789d81 100644 --- a/.github/workflows/macos-homebrew-build.yml +++ b/.github/workflows/macos-homebrew-build.yml @@ -26,6 +26,7 @@ jobs: run: brew install ccache cmake + coreutils llvm qhull yajl @@ -114,9 +115,11 @@ jobs: hdiutil attach Talipot*.dmg cp -r /Volumes/Talipot*/Talipot*.app /Applications/ sudo xattr -r -d com.apple.quarantine /Applications/Talipot*.app + rm -rf ~/.Talipot* /Applications/Talipot*.app/Contents/MacOS/Talipot \ --check-application-starts \ --debug-plugins-load + /Users/runner/.Talipot-1.0/venv3.13/bin/python3 -m ensurepip --upgrade --default-pip hdiutil detach /Volumes/Talipot* - name: Upload Talipot bundle to GitHub Actions artifacts uses: actions/upload-artifact@v4 diff --git a/.github/workflows/macos-macports-build.yml b/.github/workflows/macos-macports-build.yml index e0ce11e1ae..f36da54488 100644 --- a/.github/workflows/macos-macports-build.yml +++ b/.github/workflows/macos-macports-build.yml @@ -50,6 +50,7 @@ jobs: cmake clang-${{ env.CLANG_VERSION }} ccache + coreutils zlib qhull yajl @@ -125,9 +126,11 @@ jobs: hdiutil attach Talipot*.dmg cp -r /Volumes/Talipot*/Talipot*.app /Applications/ sudo xattr -r -d com.apple.quarantine /Applications/Talipot*.app + rm -rf ~/.Talipot* /Applications/Talipot*.app/Contents/MacOS/Talipot \ --check-application-starts \ --debug-plugins-load + /Users/runner/.Talipot-1.0/venv3.13/bin/python3 -m ensurepip --upgrade --default-pip hdiutil detach /Volumes/Talipot* - name: Upload Talipot bundle to GitHub Actions artifacts uses: actions/upload-artifact@v4 diff --git a/.github/workflows/windows-mingw64-build.yml b/.github/workflows/windows-mingw64-build.yml index 21323e1c1b..8f64d2f4a5 100644 --- a/.github/workflows/windows-mingw64-build.yml +++ b/.github/workflows/windows-mingw64-build.yml @@ -7,6 +7,8 @@ jobs: mingw64: name: ${{ matrix.config.name }} runs-on: windows-latest + env: + PYTHON_VERSION: "3.13" defaults: run: shell: msys2 {0} @@ -40,7 +42,6 @@ jobs: mingw-w64-${{ matrix.config.arch }}-qhull mingw-w64-${{ matrix.config.arch }}-graphviz mingw-w64-${{ matrix.config.arch }}-libgit2 - mingw-w64-${{ matrix.config.arch }}-python mingw-w64-${{ matrix.config.arch }}-cppunit mingw-w64-${{ matrix.config.arch }}-fontconfig mingw-w64-${{ matrix.config.arch }}-freetype @@ -48,8 +49,16 @@ jobs: mingw-w64-${{ matrix.config.arch }}-glew mingw-w64-${{ matrix.config.arch }}-qt5 mingw-w64-${{ matrix.config.arch }}-quazip - mingw-w64-${{ matrix.config.arch }}-python-sphinx - mingw-w64-${{ matrix.config.arch }}-sip + - name: Install Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v5 + id: python-install + with: + python-version: "${{ env.PYTHON_VERSION }}" + - name: Install sip and sphinx + run: | + set PATH=%Python3_ROOT_DIR%\Scripts:%PATH% + pip install sip sphinx + shell: cmd - name: Prepare ccache timestamp id: get-current-date run: | @@ -75,7 +84,7 @@ jobs: -DCMAKE_BUILD_TYPE=Release -DCMAKE_NEED_RESPONSE=ON -DCMAKE_INSTALL_PREFIX=$PWD/install - -DPython3_EXECUTABLE=/${{ matrix.config.msystem }}/bin/python3 + -DPython3_EXECUTABLE=$Python3_ROOT_DIR/python.exe -DTALIPOT_BUILD_TESTS=ON -DTALIPOT_USE_CCACHE=ON .. - name: Talipot build diff --git a/bundlers/linux/make_appimage_bundle.sh.in b/bundlers/linux/make_appimage_bundle.sh.in index a8591840b4..5d0537dffd 100644 --- a/bundlers/linux/make_appimage_bundle.sh.in +++ b/bundlers/linux/make_appimage_bundle.sh.in @@ -103,6 +103,7 @@ export LD_LIBRARY_PATH=${QT_INSTALL_LIBS_DIR}:${LD_LIBRARY_PATH} export LD_LIBRARY_PATH=$(dirname @PYTHON_LIBRARY@):$LD_LIBRARY_PATH # add Python environment +cp -v -p $TALIPOT_INSTALL_DIR/bin/python3* $BUNDLE_BIN_DIR rm -f $BUNDLE_LIB_DIR/libpython* PYTHON_LIB=$(ldd $(ls $BUNDLE_LIB_DIR/libtalipot-python*) | \ grep libpython | awk '{print $3}') @@ -125,9 +126,16 @@ if [ "$PYTHON_LIB" != "" ]; then fi echo "copying $(dirname $PYTHON_LIB)/$PYTHON_PACKAGE_DIR files into \ $PYTHON_PACKAGE_BUNDLE_DIR" - find . \( -type f \) \( ! -name "*.pyc" \) \( ! -name "*.pyo" \) -exec \ + find . \( -type f \) \( ! -name "*.pyo" \) -exec \ cp --parents --preserve=mode {} $PYTHON_PACKAGE_BUNDLE_DIR \; popd > /dev/null 2>&1 + mkdir -p $PYTHON_PACKAGE_BUNDLE_DIR/ensurepip/_bundled + pushd $PYTHON_PACKAGE_BUNDLE_DIR/ensurepip/_bundled > /dev/null 2>&1 + setuptools_version=$(python3.11 -c "import ensurepip;print(ensurepip._SETUPTOOLS_VERSION)") + pip_version=$(python3.11 -c "import ensurepip;print(ensurepip._PIP_VERSION)") + wget https://pypi.debian.net/setuptools/setuptools-${setuptools_version}-py3-none-any.whl + wget https://pypi.debian.net/pip/pip-${pip_version}-py3-none-any.whl + popd > /dev/null 2>&1 fi # copy required shared libs using linuxdeployqt tool diff --git a/bundlers/mac/CMakeLists.txt b/bundlers/mac/CMakeLists.txt index 75f782979d..f4821097b1 100644 --- a/bundlers/mac/CMakeLists.txt +++ b/bundlers/mac/CMakeLists.txt @@ -4,5 +4,5 @@ CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/mac_bundle.sh.in" ADD_CUSTOM_TARGET( bundle COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_install.cmake - COMMAND sh mac_bundle.sh ${CMAKE_BINARY_DIR} + COMMAND bash mac_bundle.sh ${CMAKE_BINARY_DIR} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/bundlers/mac/mac_bundle.sh.in b/bundlers/mac/mac_bundle.sh.in index a514f93c95..8411d90d56 100755 --- a/bundlers/mac/mac_bundle.sh.in +++ b/bundlers/mac/mac_bundle.sh.in @@ -67,8 +67,10 @@ cp -r $GV_DIR Frameworks/graphviz echo 'Copying Python Framework' mkdir -p Frameworks/Python.framework/Versions/@PYTHON_VERSION@ cp -r @PYTHON_STDLIB_DIR@/../../* Frameworks/Python.framework/Versions/@PYTHON_VERSION@/ -find Frameworks/Python.framework/ | grep -E "(__pycache__|\.pyc|\.pyo$)" | xargs rm -rf +find Frameworks/Python.framework/ | grep -E "(__pycache__|\.pyo$)" | xargs rm -rf rm -rf Frameworks/Python.framework/Versions/@PYTHON_VERSION@/share +rm -f bin/python3* +ln -s ../Frameworks/Python.framework/Versions/@PYTHON_VERSION@/bin/python3 bin/python3 echo 'Copying License' cd "${DEST_DIR}/application" @@ -76,11 +78,13 @@ cp "${SRC_DIR}/../../LICENSE" . QT_LIB_DIR="@QT_QTCORE_LIBRARY@/.." TALIPOT_APP=${DEST_DIR}/application/Talipot.app -echo 'Copying Resources' + +echo 'Copying Qt Resources' cd "${TALIPOT_APP}/Contents/Frameworks/" cp -r "${QT_LIB_DIR}/QtGui.framework/Resources/qt_menu.nib" ../Resources 2>/dev/null cp "@QT_QTCLUCENE_LIBRARY@" . 2>/dev/null +echo 'Executing macdeployqt' cd .. # configure talipot mv bin/talipot MacOS/Talipot @@ -88,6 +92,7 @@ mv bin/talipot MacOS/Talipot # its extension is .so instead of .dylib) talipot_python_module=$(ls ${TALIPOT_APP}/Contents/lib/talipot/python/talipot/native/talipot*.so) mac_deploy_qt_opts=-executable=${talipot_python_module} + if [ $(echo ${QT_VERSION} | cut -c1) -ge 6 ] then # qt plugins end up with broken rpaths when using macdeployt from Qt6 @@ -100,7 +105,7 @@ ${QT_BINARY_DIR}/macdeployqt ${TALIPOT_APP} $mac_deploy_qt_opts # ensure clang libc++* are present in bundle LIB_CXX_DIR=$(echo "@CMAKE_SHARED_LINKER_FLAGS@" | cut -f1 -d" " | cut -c3-) -if [ -d ${LIB_CXX_DIR} ] +if [ -f ${LIB_CXX_DIR}/libc++.1.dylib ] then cp ${LIB_CXX_DIR}/libc++.1.dylib ${TALIPOT_APP}/Contents/Frameworks/ cp ${LIB_CXX_DIR}/libc++abi.1.dylib ${TALIPOT_APP}/Contents/Frameworks/ @@ -127,6 +132,38 @@ platforms position styles tls done fi +echo 'Fix remaining hardcoded dylib loading paths' +# fix remaining hardcoded dylib loading paths in binaries to ensure bundle portability +pushd "${DEST_DIR}/application/Talipot.app/Contents/" > /dev/null 2>&1 +export PATH=/opt/local/libexec/gnubin:$PATH +realpath_cmd=$(which grealpath || which realpath) +for binary in $(find . -perm +0111 -type f) +do + for dylib in $(otool -L $binary | \ + grep -E '/opt/local/|/usr/local/|/opt/homebrew/' | cut -d '(' -s -f 1 | xargs) + do + for pattern in /Frameworks/ /lib/ + do + before_pattern=${dylib%%"$pattern"*} + if [ "$before_pattern" != "$dylib" ] + then + let pos=${#before_pattern}+${#pattern} + dylib_subpath=${dylib:${pos}} + + binary_dir=$(dirname $binary) + rel_path=$($realpath_cmd --relative-to=$binary_dir ./Frameworks/$dylib_subpath) + loader_path="@loader_path/$rel_path" + + echo "install_name_tool -change $dylib $loader_path $binary" + install_name_tool -change $dylib $loader_path $binary + + break + fi + done + done +done +popd > /dev/null 2>&1 + mv MacOS/Talipot bin/talipot # rename mv ${TALIPOT_APP} ${DEST_DIR}/application/${APP_NAME}-@TalipotVersion@.app diff --git a/library/talipot-python/include/talipot/PythonInterpreter.h b/library/talipot-python/include/talipot/PythonInterpreter.h index fe93fba14c..cdd12171ca 100644 --- a/library/talipot-python/include/talipot/PythonInterpreter.h +++ b/library/talipot-python/include/talipot/PythonInterpreter.h @@ -1,6 +1,6 @@ /** * - * Copyright (C) 2019-2021 The Talipot developers + * Copyright (C) 2019-2024 The Talipot developers * * Talipot is a fork of Tulip, created by David Auber * and the Tulip development Team from LaBRI, University of Bordeaux @@ -51,6 +51,7 @@ class TLP_PYTHON_SCOPE PythonInterpreter : public QObject, public Singleton #endif +#include #include #include @@ -133,8 +134,10 @@ int tracefunc(PyObject *, PyFrameObject *, int what, PyObject *) { const QString PythonInterpreter::pythonPluginsPath(tlpStringToQString(tlp::TalipotLibDir) + "talipot/python/"); -const QString PythonInterpreter::pythonPluginsPathHome(QDir::homePath() + "/.Talipot-" + - TALIPOT_MM_VERSION + "/plugins/python"); +const QString talipotUserDirectory = QDir::homePath() + "/.Talipot-" + TALIPOT_MM_VERSION; +const QString talipotVenvDirectory = + talipotUserDirectory + "/venv" + PythonVersionChecker::compiledVersion(); +const QString PythonInterpreter::pythonPluginsPathHome(talipotUserDirectory + "/plugins/python"); const char PythonInterpreter::pythonReservedCharacters[] = { '#', '%', '/', '+', '-', '&', '*', '<', '>', '|', '~', '^', '=', @@ -211,6 +214,10 @@ PythonInterpreter::PythonInterpreter() #endif } +#ifndef MSYS2_PYTHON + setupVirtualEnv(); +#endif + holdGIL(); importModule("sys"); @@ -305,6 +312,61 @@ PythonInterpreter::PythonInterpreter() } releaseGIL(); + +#ifndef MSYS2_PYTHON + setupVirtualEnv(); +#endif +} + +void PythonInterpreter::setupVirtualEnv() { +#ifdef Q_OS_WIN + if (!QFileInfo(talipotVenvDirectory + "/Scripts/pip.exe").exists()) { +#else + if (!QFileInfo(talipotVenvDirectory + "/bin/pip").exists()) { +#endif + runString(QString(R"( +import os +import platform +import sys +import venv +python_command = 'python3' +if platform.system() == 'Windows': + python_command = 'python.exe' +sys._base_executable = os.path.join('%1', python_command) +os.environ['LD_LIBRARY_PATH'] = '%2'; +os.environ['DYLD_FALLBACK_LIBRARY_PATH'] = '%2'; +os.environ['DYLD_FRAMEWORK_PATH'] = '%2'; +venv.create('%3', with_pip=False, symlinks=True) +)") + .arg(QApplication::applicationDirPath(), tlpStringToQString(tlp::TalipotLibDir), + talipotVenvDirectory)); + } + + runString(QString(R"( +import os +import platform +import sys + +base = '%1' +if platform.system() == 'Windows': + site_packages = os.path.join(base, 'Lib', 'site-packages') +else: + site_packages = os.path.join( + base, 'lib', + 'python%s.%s' % (sys.version_info.major, sys.version_info.minor), + 'site-packages') +prev_sys_path = list(sys.path) +import site +site.addsitedir(site_packages) +sys.real_prefix = sys.prefix +sys.prefix = base +new_sys_path = [] +for item in list(sys.path): + if item not in prev_sys_path: + new_sys_path.append(item) + sys.path.remove(item) +sys.path[:0] = new_sys_path)") + .arg(talipotVenvDirectory)); } PythonInterpreter::~PythonInterpreter() { diff --git a/software/talipot/CMakeLists.txt b/software/talipot/CMakeLists.txt index 4dd66b879d..37c14699c8 100644 --- a/software/talipot/CMakeLists.txt +++ b/software/talipot/CMakeLists.txt @@ -269,3 +269,23 @@ IF(LINUX) OUTPUT_QUIET ERROR_QUIET)") ENDIF(TALIPOT_LINUX_DESKTOP_REGISTRATION) ENDIF(LINUX) + +IF(NOT MSYS2_PYTHON) + STRING(REPLACE "\\" "/" PYTHON_EXE_PATH "${PYTHON_EXECUTABLE}") + GET_FILENAME_COMPONENT(PYTHON_EXE_NAME "${PYTHON_EXECUTABLE}" NAME) + + INSTALL( + CODE " +FILE(COPY \"${PYTHON_EXE_PATH}\" + DESTINATION \"\${CMAKE_INSTALL_PREFIX}/bin/\" FOLLOW_SYMLINK_CHAIN) +IF(NOT WIN32 + AND NOT ${PYTHON_EXE_NAME} STREQUAL python3 + AND NOT EXISTS ${CMAKE_INSTALL_PREFIX}/bin/python3) + FILE(CREATE_LINK ${CMAKE_INSTALL_PREFIX}/bin/${PYTHON_EXE_NAME} + ${CMAKE_INSTALL_PREFIX}/bin/python3 SYMBOLIC) +ENDIF( + NOT WIN32 + AND NOT ${PYTHON_EXE_NAME} STREQUAL python3 + AND NOT EXISTS ${CMAKE_INSTALL_PREFIX}/bin/python3) +") +ENDIF(NOT MSYS2_PYTHON)