From 3bb1c3a4dd3605c9504afbd1fd5c6f4f5c3b95ed Mon Sep 17 00:00:00 2001 From: Yuto Ashida Date: Mon, 8 Aug 2022 01:56:06 +0900 Subject: [PATCH 1/9] =?UTF-8?q?Windows=E3=83=93=E3=83=AB=E3=83=89=E3=82=92?= =?UTF-8?q?PyInstaller=E3=81=B8=E7=A7=BB=E8=A1=8C=20(#443)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit use pyinstaller in github action for windows build --- .github/workflows/build.yml | 85 ++++++------------------------------- generate_licenses.py | 9 +++- requirements-dev.in | 2 +- requirements-dev.txt | 26 ++++++++++-- 4 files changed, 45 insertions(+), 77 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a1cac06e2..0b682b68c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -379,18 +379,14 @@ jobs: architecture: "x64" voicevox_core_asset_prefix: voicevox_core-windows-x64-cpu onnxruntime_url: https://github.com/microsoft/onnxruntime/releases/download/v1.10.0/onnxruntime-win-x64-1.10.0.zip - ccache_url: https://github.com/ccache/ccache/releases/download/v4.4.1/ccache-4.4.1-windows-64.zip artifact_name: windows-cpu - nuitka_cache_path: nuitka_cache # Windows DirectML - os: windows-2019 architecture: "x64" voicevox_core_asset_prefix: voicevox_core-windows-x64-directml onnxruntime_url: https://github.com/microsoft/onnxruntime/releases/download/v1.10.0/Microsoft.ML.OnnxRuntime.DirectML.1.10.0.zip directml_url: https://www.nuget.org/api/v2/package/Microsoft.AI.DirectML/1.8.0 - ccache_url: https://github.com/ccache/ccache/releases/download/v4.4.1/ccache-4.4.1-windows-64.zip artifact_name: windows-directml - nuitka_cache_path: nuitka_cache # Windows NVIDIA GPU - os: windows-2019 architecture: "x64" @@ -398,9 +394,7 @@ jobs: onnxruntime_url: https://github.com/microsoft/onnxruntime/releases/download/v1.10.0/onnxruntime-win-x64-gpu-1.10.0.zip cuda_version: "11.4.2" cudnn_url: https://developer.download.nvidia.com/compute/redist/cudnn/v8.2.4/cudnn-11.4-windows-x64-v8.2.4.15.zip - ccache_url: https://github.com/ccache/ccache/releases/download/v4.4.1/ccache-4.4.1-windows-64.zip artifact_name: windows-nvidia - nuitka_cache_path: nuitka_cache runs-on: ${{ matrix.os }} @@ -501,9 +495,6 @@ jobs: - name: Install Python dependencies shell: bash run: | - # FIXME: Nuitka cannot build with setuptools>=60.7.0 - # https://github.com/Nuitka/Nuitka/issues/1406 - python -m pip install --upgrade pip setuptools==60.6.0 wheel python -m pip install -r requirements-dev.txt # Download pyopenjtalk dictionary @@ -527,32 +518,6 @@ jobs: shell: bash run: mkdir -p download/ - # Install Ccache - - name: Export Ccache url to calc hash - shell: bash - run: echo "${{ matrix.ccache_url }}" > download/ccache_url.txt - - - name: Prepare Ccache - uses: actions/cache@v2 - id: ccache-cache - with: - key: ${{ matrix.os }}-ccache-${{ hashFiles('download/ccache_url.txt') }} - path: download/ccache - - - name: Download Ccache - if: steps.ccache-cache.outputs.cache-hit != 'true' - shell: bash - run: | - curl -L "${{ matrix.ccache_url }}" > download/ccache.zip - unzip download/ccache.zip -d download/ - rm download/ccache.zip - mv download/ccache-*/ download/ccache - - - name: Install Ccache - shell: bash - run: | - echo "$HOME/download/ccache" >> $GITHUB_PATH - # Donwload DirectML - name: Export DirectML url to calc hash if: endswith(matrix.artifact_name, '-directml') @@ -672,52 +637,30 @@ jobs: # FIXME: VOICEVOX (editor) cannot build without licenses.json cp engine_manifest_assets/dependency_licenses.json licenses.json - - name: Cache Nuitka (ccache, module-cache) - uses: actions/cache@v2 - id: nuitka-cache - with: - path: ${{ matrix.nuitka_cache_path }} - key: ${{ runner.os }}-nuitka-${{ matrix.artifact_name }}-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-nuitka-${{ matrix.artifact_name }}- - - name: Show disk space (debug info) shell: bash run: | df -h - - name: Build run.py with Nuitka + - name: Build run.py with PyInstaller shell: bash - env: - NUITKA_CACHE_DIR: ${{ matrix.nuitka_cache_path }} run: | set -eux # Replace version sed -i "s/__version__ = \"latest\"/__version__ = \"${{ env.VOICEVOX_ENGINE_VERSION }}\"/" voicevox_engine/__init__.py - python -m nuitka \ - --standalone \ - --assume-yes-for-downloads \ - --plugin-enable=numpy \ - --plugin-enable=multiprocessing \ - --follow-import-to=numpy \ - --follow-import-to=aiofiles \ - --include-package=uvicorn \ - --include-package=anyio \ - --include-package-data=pyopenjtalk \ - --include-package-data=scipy \ - --include-data-file="licenses.json=./" \ - --include-data-file="presets.yaml=./" \ - --include-data-file="default.csv=./" \ - --include-data-file="engine_manifest.json=./" \ - --include-data-file="download/onnxruntime/lib/onnxruntime.dll=./" \ - --include-data-file="download/core/core.dll=./" \ - --include-data-dir="speaker_info=./speaker_info" \ - --include-data-dir="engine_manifest_assets=./engine_manifest_assets" \ - --msvc=14.2 \ - --follow-imports \ - --no-prefer-source-code \ + pyinstaller \ + --noconfirm \ + --collect-data pyopenjtalk \ + --add-data "licenses.json;." \ + --add-data "presets.yaml;." \ + --add-data "default.csv;." \ + --add-data "engine_manifest.json;." \ + --add-data "speaker_info;speaker_info" \ + --add-data "engine_manifest_assets;engine_manifest_assets" \ + --add-binary "download/onnxruntime/lib/onnxruntime.dll;." \ + --add-binary "download/core/core.dll;." \ run.py - name: Show disk space (debug info) @@ -737,12 +680,12 @@ jobs: # Build artifact directory mkdir -p artifact - ln -sf "$(pwd)/run.dist"/* artifact/ + ln -sf "$(pwd)/dist/run"/* artifact/ # Copy DLL dependencies if [ -f "download/onnxruntime/lib/onnxruntime_providers_cuda.dll" ]; then - # ONNX Runtime providers (Nuitka does not copy dynamic loaded libraries) + # ONNX Runtime providers (PyInstaller does not copy dynamic loaded libraries) ln -sf "$(pwd)/download/onnxruntime/lib"/onnxruntime_*.dll artifact/ # CUDA diff --git a/generate_licenses.py b/generate_licenses.py index 133fff44b..3ef10abf2 100644 --- a/generate_licenses.py +++ b/generate_licenses.py @@ -161,9 +161,14 @@ def generate_licenses() -> List[License]: if license.text == "UNKNOWN": if license.name.lower() == "core" and license.version == "0.0.0": continue - elif license.name.lower() == "nuitka": + elif license.name.lower() == "future": with urllib.request.urlopen( - "https://raw.githubusercontent.com/Nuitka/Nuitka/develop/LICENSE.txt" + "https://raw.githubusercontent.com/PythonCharmers/python-future/master/LICENSE.txt" # noqa: B950 + ) as res: + license.text = res.read().decode() + elif license.name.lower() == "pefile": + with urllib.request.urlopen( + "https://raw.githubusercontent.com/erocarrera/pefile/master/LICENSE" # noqa: B950 ) as res: license.text = res.read().decode() elif license.name.lower() == "pyopenjtalk": diff --git a/requirements-dev.in b/requirements-dev.in index 0719d79f2..6e8836b30 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -1,6 +1,6 @@ -r requirements.in cython -nuitka +pyinstaller pip-licenses pip-tools pre-commit diff --git a/requirements-dev.txt b/requirements-dev.txt index 8d744799c..c39311e01 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,6 +6,8 @@ # aiofiles==0.7.0 # via -r requirements.in +altgraph==0.17.2 + # via pyinstaller anyio==3.3.4 # via starlette appdirs==1.4.4 @@ -16,10 +18,14 @@ atomicwrites==1.4.0 # via -r requirements-dev.in backports.entry-points-selectable==1.1.1 # via virtualenv +certifi==2022.6.15 + # via requests cffi==1.15.0 # via soundfile cfgv==3.3.1 # via pre-commit +charset-normalizer==2.1.0 + # via requests click==8.0.3 # via # pip-tools @@ -40,22 +46,26 @@ fastapi==0.70.0 # via -r requirements.in filelock==3.4.0 # via virtualenv +future==0.18.2 + # via pefile h11==0.12.0 # via uvicorn identify==2.4.0 # via pre-commit idna==3.3 - # via anyio + # via + # anyio + # requests nodeenv==1.6.0 # via pre-commit -nuitka==0.6.17.4 - # via -r requirements-dev.in numpy==1.20.0 # via # -r requirements.in # pyopenjtalk # pyworld # scipy +pefile==2022.5.30 + # via pyinstaller pep517==0.12.0 # via pip-tools pip-licenses==3.5.3 @@ -72,16 +82,24 @@ pycparser==2.20 # via cffi pydantic==1.8.2 # via fastapi +pyinstaller==5.3 + # via -r requirements-dev.in +pyinstaller-hooks-contrib==2022.8 + # via pyinstaller pyopenjtalk @ git+https://github.com/VOICEVOX/pyopenjtalk@50b0296a9e1b666e5a09a41ec9e9284a2a9b608f # via -r requirements.in python-multipart==0.0.5 # via -r requirements.in +pywin32-ctypes==0.2.0 + # via pyinstaller pyworld==0.3.0 # via -r requirements.in pyyaml==6.0 # via # -r requirements.in # pre-commit +requests==2.28.1 + # via -r requirements.in scipy==1.7.1 # via -r requirements.in six==1.16.0 @@ -103,6 +121,8 @@ tqdm==4.62.3 # via pyopenjtalk typing-extensions==3.10.0.2 # via pydantic +urllib3==1.26.11 + # via requests uvicorn==0.15.0 # via -r requirements.in virtualenv==20.10.0 From be269516ad26fc740cc87ccadcf35f109d4b46ce Mon Sep 17 00:00:00 2001 From: Yuto Ashida Date: Sat, 13 Aug 2022 01:38:22 +0900 Subject: [PATCH 2/9] =?UTF-8?q?Mac=E3=83=93=E3=83=AB=E3=83=89=E3=82=92PyIn?= =?UTF-8?q?staller=E3=81=B8=E7=A7=BB=E8=A1=8C=E3=81=97=E3=80=81Windows?= =?UTF-8?q?=E3=81=A8=E3=83=93=E3=83=AB=E3=83=89=E3=82=B8=E3=83=A7=E3=83=96?= =?UTF-8?q?=E3=82=92=E7=B5=B1=E5=90=88=E3=81=99=E3=82=8B=20(#446)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * use pyinstaller in github action for mac build and integrate windows and mac build process * use spec file in github actions * fix folder name * use noconfirm option * remove build utils of macos * fix miss and split step * integrate steps * add comment to run.spec --- .github/workflows/build.yml | 310 +++++------------- .gitignore | 6 - build_util/macos/build_util_macos/__init__.py | 0 .../macos/build_util_macos/shlib_tools.py | 93 ------ build_util/macos/copy_missing_dylibs.py | 54 --- build_util/macos/fix_rpaths.py | 67 ---- run.spec | 88 +++++ 7 files changed, 161 insertions(+), 457 deletions(-) delete mode 100644 build_util/macos/build_util_macos/__init__.py delete mode 100644 build_util/macos/build_util_macos/shlib_tools.py delete mode 100644 build_util/macos/copy_missing_dylibs.py delete mode 100644 build_util/macos/fix_rpaths.py create mode 100644 run.spec diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0b682b68c..c3dd7f71f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,217 +29,6 @@ env: ${{ github.event.release.tag_name || github.event.inputs.version || 'latest' }} jobs: - # Build Mac binary (x64 arch only) - build-mac: - strategy: - matrix: - include: - - os: macos-11 - python_architecture: "x64" - voicevox_core_asset_prefix: voicevox_core-osx-universal2-cpu - onnxruntime_url: https://github.com/microsoft/onnxruntime/releases/download/v1.10.0/onnxruntime-osx-universal2-1.10.0.tgz - artifact_name: macos-x64 - - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v2 - - - name: Install CCache - shell: bash - run: brew install ccache - - # NOTE: The default sed of macOS is BSD sed. - # There is a difference in specification between BSD sed and GNU sed, - # so you need to install GNU sed. - - name: Install GNU sed on macOS - shell: bash - run: brew install gnu-sed - - - name: Create download and build directory - run: mkdir download build - - - name: Setup Python - id: setup-python - uses: actions/setup-python@v2 - with: - python-version: ${{ env.PYTHON_VERSION }} - architecture: ${{ matrix.python_architecture }} - cache: pip - - - name: Install Python dependencies - shell: bash - run: | - # FIXME: Nuitka cannot build with setuptools>=60.7.0 - # https://github.com/Nuitka/Nuitka/issues/1406 - pip install --upgrade pip setuptools==60.6.0 wheel - pip install -r requirements-dev.txt - - - name: Generate licenses.json - shell: bash - run: | - python generate_licenses.py > engine_manifest_assets/dependency_licenses.json - # FIXME: VOICEVOX (editor) cannot build without licenses.json - cp engine_manifest_assets/dependency_licenses.json licenses.json - - # Download ONNX Runtime - - name: Export ONNX Runtime url to calc hash - shell: bash - run: echo "${{ matrix.onnxruntime_url }}" > download/onnxruntime_url.txt - - - name: Prepare ONNX Runtime cache - uses: actions/cache@v2 - id: onnxruntime-dylib-cache - with: - key: ${{ matrix.os }}-onnxruntime-dylib-${{ hashFiles('download/onnxruntime_url.txt') }}-v1 - path: download/onnxruntime - - - name: Download ONNX Runtime - if: steps.onnxruntime-dylib-cache.outputs.cache-hit != 'true' - shell: bash - run: | - curl -L "${{ matrix.onnxruntime_url }}" > download/onnxruntime.tgz - - # extract only dylibs - mkdir -p download/onnxruntime - tar xf "download/onnxruntime.tgz" -C "download/onnxruntime" --strip-components 1 'onnxruntime-*/lib/*.dylib' - rm download/onnxruntime.tgz - - - name: Show disk space (debug info) - shell: bash - run: | - df -h - - # Download VOICEVOX RESOURCE - - name: Prepare VOICEVOX RESOURCE cache - uses: actions/cache@v2 - id: voicevox-resource-cache - with: - key: voicevox-resource-${{ env.VOICEVOX_RESOURCE_VERSION }} - path: download/resource - - - name: Checkout VOICEVOX RESOURCE - if: steps.voicevox-resource-cache.outputs.cache-hit != 'true' - uses: actions/checkout@v2 - with: - repository: VOICEVOX/voicevox_resource - ref: ${{ env.VOICEVOX_RESOURCE_VERSION }} - path: download/resource - - # Merge VOICEVOX RESOURCE - - name: Merge VOICEVOX RESOURCE - shell: bash - env: - DOWNLOAD_RESOURCE_PATH: download/resource - run: bash build_util/merge_voicevox_resource.bash - - - name: Prepare VOICEVOX Core release cache - uses: actions/cache@v2 - id: voicevox-core-cache - with: - key: ${{ matrix.os }}-voicevox-core-${{ matrix.voicevox_core_asset_prefix }}-${{ env.VOICEVOX_CORE_VERSION }} - path: download/core - - - name: Download VOICEVOX Core release - if: steps.voicevox-core-cache.outputs.cache-hit != 'true' - shell: bash - env: - VOICEVOX_CORE_ASSET_NAME: ${{ matrix.voicevox_core_asset_prefix }}-${{ env.VOICEVOX_CORE_VERSION }} - run: | - # extracted like download/core/libcore.dylib - curl -L "https://github.com/VOICEVOX/voicevox_core/releases/download/${{ env.VOICEVOX_CORE_VERSION }}/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip" > download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip - ditto -x -k --sequesterRsrc --rsrc download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip download/ - mkdir -p download/core - mv download/${{ env.VOICEVOX_CORE_ASSET_NAME }}/* download/core - rm -rf download/${{ env.VOICEVOX_CORE_ASSET_NAME }} - rm download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip - - - name: Download PyOpenJTalk dictionary - shell: bash - run: | - set -eux - - # Download pyopenjtalk dictionary - # try 5 times, sleep 5 seconds before retry - for i in $(seq 5); do - EXIT_CODE=0 - python -c "import pyopenjtalk; pyopenjtalk._lazy_init()" || EXIT_CODE=$? - - if [ "$EXIT_CODE" = "0" ]; then - break - fi - - sleep 5 - done - - if [ "$EXIT_CODE" != "0" ]; then - exit "$EXIT_CODE" - fi - - - name: Build run.py - run: | - set -eux - - # Replace version - gsed -i "s/__version__ = \"latest\"/__version__ = \"${{ env.VOICEVOX_ENGINE_VERSION }}\"/" ../voicevox_engine/__init__.py - - # Nuitka does not copy grandchild dependencies. Explicitly copy libonnxruntime - python -m nuitka \ - --output-dir=./ \ - --standalone \ - --plugin-enable=numpy \ - --plugin-enable=multiprocessing \ - --follow-import-to=numpy \ - --follow-import-to=aiofiles \ - --include-package=uvicorn \ - --include-package=anyio \ - --include-package-data=pyopenjtalk \ - --include-package-data=scipy \ - --include-data-file=../licenses.json=./ \ - --include-data-file=../presets.yaml=./ \ - --include-data-file=../default.csv=./ \ - --include-data-file=../engine_manifest.json=./ \ - --include-data-file=../download/core/libcore.dylib=./ \ - --include-data-file=../download/onnxruntime/lib/libonnxruntime.dylib=./ \ - --include-data-file=${{ env.pythonLocation }}/lib/python*/site-packages/scipy/.dylibs/*.dylib=./scipy/.dylibs/ \ - --include-data-file=${{ env.pythonLocation }}/lib/python*/site-packages/_soundfile_data/*=./_soundfile_data/ \ - --include-data-dir=../speaker_info=./speaker_info \ - --include-data-dir=../engine_manifest_assets=./engine_manifest_assets \ - --follow-imports \ - --no-prefer-source-code \ - ../run.py - working-directory: build - - - name: Set @rpath to @executable_path - run: | - install_name_tool -add_rpath @executable_path/. build/run.dist/run - - # NOTE: This task should ideally be done by Nuitka in the `Build run.py` step. - # Please remove this step when you have solved the problem with Nuitka. - - name: Copy the missing .dylib files into the distribution - run: | - python build_util/macos/copy_missing_dylibs.py build/run.dist/ - - # NOTE: This task should ideally be done by Nuitka in the `Build run.py` step. - # Please remove this step when you have solved the problem with Nuitka. - - name: Fix the rpaths of the .dylib files in the distribution - run: | - python build_util/macos/fix_rpaths.py build/run.dist/ - - # FIXME: versioned name may be useful; but - # actions/download-artifact and dawidd6/download-artifact do not support - # wildcard / forward-matching yet. - # Currently, It is good to use static artifact name for future binary test workflow. - # https://github.com/actions/toolkit/blob/ea81280a4d48fb0308d40f8f12ae00d117f8acb9/packages/artifact/src/internal/artifact-client.ts#L147 - # https://github.com/dawidd6/action-download-artifact/blob/af92a8455a59214b7b932932f2662fdefbd78126/main.js#L113 - - uses: actions/upload-artifact@v2 - # env: - # VERSIONED_ARTIFACT_NAME: | - # ${{ format('{0}-{1}', matrix.artifact_name, (env.VOICEVOX_ENGINE_VERSION != 'latest' && env.VOICEVOX_ENGINE_VERSION) || github.sha) }} - with: - name: ${{ matrix.artifact_name }} - path: build/run.dist/ - # Build Linux binary (push only buildcache image) build-linux: strategy: @@ -369,7 +158,7 @@ jobs: name: ${{ matrix.artifact_name }} path: build/run.dist/ - build-windows: + build-windows-and-mac: environment: ${{ github.event.inputs.code_signing == 'true' && 'code_signing' }} # コード署名用のenvironment strategy: matrix: @@ -395,6 +184,12 @@ jobs: cuda_version: "11.4.2" cudnn_url: https://developer.download.nvidia.com/compute/redist/cudnn/v8.2.4/cudnn-11.4-windows-x64-v8.2.4.15.zip artifact_name: windows-nvidia + # Mac CPU (x64 arch only) + - os: macos-11 + architecture: "x64" + voicevox_core_asset_prefix: voicevox_core-osx-universal2-cpu + onnxruntime_url: https://github.com/microsoft/onnxruntime/releases/download/v1.10.0/onnxruntime-osx-universal2-1.10.0.tgz + artifact_name: macos-x64 runs-on: ${{ matrix.os }} @@ -406,6 +201,14 @@ jobs: run: | df -h + # NOTE: The default sed of macOS is BSD sed. + # There is a difference in specification between BSD sed and GNU sed, + # so you need to install GNU sed. + - name: Install GNU sed on macOS + if: startsWith(matrix.os, 'macos-') + shell: bash + run: brew install gnu-sed + # Download CUDA - name: Prepare CUDA DLL cache if: matrix.cuda_version != '' @@ -481,9 +284,10 @@ jobs: df -h - name: Setup MSVC + if: startsWith(matrix.os, 'windows-') uses: ilammy/msvc-dev-cmd@v1 - # Python install path: C:/hostedtoolcache/windows/Python + # Python install path of windows: C:/hostedtoolcache/windows/Python - name: Setup Python id: setup-python uses: actions/setup-python@v2 @@ -552,13 +356,13 @@ jobs: - name: Prepare ONNX Runtime cache uses: actions/cache@v2 - id: onnxruntime-dll-cache + id: onnxruntime-cache with: - key: ${{ matrix.os }}-onnxruntime-dll-${{ hashFiles('download/onnxruntime_url.txt') }}-v1 + key: ${{ matrix.os }}-onnxruntime-${{ hashFiles('download/onnxruntime_url.txt') }}-v1 path: download/onnxruntime - - name: Download ONNX Runtime - if: steps.onnxruntime-dll-cache.outputs.cache-hit != 'true' + - name: Download ONNX Runtime (Windows) + if: steps.onnxruntime-cache.outputs.cache-hit != 'true' && startsWith(matrix.os, 'windows-') shell: bash run: | curl -L "${{ matrix.onnxruntime_url }}" > download/onnxruntime.zip @@ -576,6 +380,17 @@ jobs: rm download/onnxruntime.zip + - name: Download ONNX Runtime (Mac) + if: steps.onnxruntime-cache.outputs.cache-hit != 'true' && startsWith(matrix.os, 'macos-') + shell: bash + run: | + curl -L "${{ matrix.onnxruntime_url }}" > download/onnxruntime.tgz + + # extract only dylibs + mkdir -p download/onnxruntime + tar xf "download/onnxruntime.tgz" -C "download/onnxruntime" --strip-components 1 'onnxruntime-*/lib/*.dylib' + rm download/onnxruntime.tgz + - name: Show disk space (debug info) shell: bash run: | @@ -624,7 +439,11 @@ jobs: # unzip コマンドはこのような zip ファイルを解凍できるものの、終了コード 1 を報告して CI が落ちてしまう。 # 回避策として、unzip コマンドの代わりに 7z コマンドを用いて zip ファイルを解凍する。 # unzip download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip -d download/ - 7z x -o"download" download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip + if [[ ${{ matrix.os }} == windows-* ]]; then + 7z x -o"download" download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip + else + ditto -x -k --sequesterRsrc --rsrc download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip download/ + fi mkdir -p download/core mv download/${{ env.VOICEVOX_CORE_ASSET_NAME }}/* download/core rm -rf download/${{ env.VOICEVOX_CORE_ASSET_NAME }} @@ -647,21 +466,20 @@ jobs: run: | set -eux - # Replace version - sed -i "s/__version__ = \"latest\"/__version__ = \"${{ env.VOICEVOX_ENGINE_VERSION }}\"/" voicevox_engine/__init__.py - - pyinstaller \ - --noconfirm \ - --collect-data pyopenjtalk \ - --add-data "licenses.json;." \ - --add-data "presets.yaml;." \ - --add-data "default.csv;." \ - --add-data "engine_manifest.json;." \ - --add-data "speaker_info;speaker_info" \ - --add-data "engine_manifest_assets;engine_manifest_assets" \ - --add-binary "download/onnxruntime/lib/onnxruntime.dll;." \ - --add-binary "download/core/core.dll;." \ - run.py + # Replace version & specify dynamic libraries + if [[ ${{ matrix.os }} == macos-* ]]; then + gsed -i "s/__version__ = \"latest\"/__version__ = \"${{ env.VOICEVOX_ENGINE_VERSION }}\"/" voicevox_engine/__init__.py + LIBCORE_PATH=download/core/libcore.dylib + LIBONNXRUNTIME_PATH=download/onnxruntime/lib/libonnxruntime.dylib + else + sed -i "s/__version__ = \"latest\"/__version__ = \"${{ env.VOICEVOX_ENGINE_VERSION }}\"/" voicevox_engine/__init__.py + LIBCORE_PATH=download/core/core.dll + LIBONNXRUNTIME_PATH=download/onnxruntime/lib/onnxruntime.dll + fi + + LIBCORE_PATH="$LIBCORE_PATH" \ + LIBONNXRUNTIME_PATH="$LIBONNXRUNTIME_PATH" \ + pyinstaller --noconfirm run.spec - name: Show disk space (debug info) shell: bash @@ -669,6 +487,7 @@ jobs: df -h - name: Create artifact directory with symlink + if: startsWith(matrix.os, 'windows-') shell: bash env: PYTHON_SITE_PACKAGES_DIR: C:/hostedtoolcache/windows/python/${{ steps.setup-python.outputs.python-version }}/x64/lib/site-packages @@ -680,7 +499,7 @@ jobs: # Build artifact directory mkdir -p artifact - ln -sf "$(pwd)/dist/run"/* artifact/ + ln -sf "$(pwd)/dist/run.dist"/* artifact/ # Copy DLL dependencies @@ -708,8 +527,13 @@ jobs: # pysoundfile ln -sf "${{ env.PYTHON_SITE_PACKAGES_DIR }}/_soundfile_data" artifact/ + - name: Set @rpath to @executable_path + if: startsWith(matrix.os, 'macos-') + run: | + install_name_tool -add_rpath @executable_path/. dist/run.dist/run + - name: Code signing - if: github.event.inputs.code_signing == 'true' + if: github.event.inputs.code_signing == 'true' && startsWith(matrix.os, 'windows-') shell: bash run: | bash build_util/codesign.bash "artifact/run.exe" @@ -723,7 +547,9 @@ jobs: # Currently, It is good to use static artifact name for future binary test workflow. # https://github.com/actions/toolkit/blob/ea81280a4d48fb0308d40f8f12ae00d117f8acb9/packages/artifact/src/internal/artifact-client.ts#L147 # https://github.com/dawidd6/action-download-artifact/blob/af92a8455a59214b7b932932f2662fdefbd78126/main.js#L113 - - uses: actions/upload-artifact@v2 + - name: Upload artifact (Windows) + if: startsWith(matrix.os, 'windows-') + uses: actions/upload-artifact@v2 # env: # VERSIONED_ARTIFACT_NAME: | # ${{ format('{0}-{1}', matrix.artifact_name, (env.VOICEVOX_ENGINE_VERSION != 'latest' && env.VOICEVOX_ENGINE_VERSION) || github.sha) }} @@ -732,9 +558,19 @@ jobs: path: | artifact/ + - name: Upload artifact (Mac) + if: startsWith(matrix.os, 'macos-') + uses: actions/upload-artifact@v2 + # env: + # VERSIONED_ARTIFACT_NAME: | + # ${{ format('{0}-{1}', matrix.artifact_name, (env.VOICEVOX_ENGINE_VERSION != 'latest' && env.VOICEVOX_ENGINE_VERSION) || github.sha) }} + with: + name: ${{ matrix.artifact_name }} + path: dist/run.dist/ + upload-to-release: if: (github.event.release.tag_name || github.event.inputs.version) != '' - needs: [build-mac, build-linux, build-windows] + needs: [build-linux, build-windows-and-mac] runs-on: ubuntu-latest strategy: matrix: diff --git a/.gitignore b/.gitignore index 9e8eea619..6b40bcf89 100644 --- a/.gitignore +++ b/.gitignore @@ -36,12 +36,6 @@ share/python-wheels/ *.egg MANIFEST -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - # Installer logs pip-log.txt pip-delete-this-directory.txt diff --git a/build_util/macos/build_util_macos/__init__.py b/build_util/macos/build_util_macos/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/build_util/macos/build_util_macos/shlib_tools.py b/build_util/macos/build_util_macos/shlib_tools.py deleted file mode 100644 index ae53979f0..000000000 --- a/build_util/macos/build_util_macos/shlib_tools.py +++ /dev/null @@ -1,93 +0,0 @@ -""" -macOSにおいて共有ライブラリを操作するためのツールをまとめたモジュール -""" - -import subprocess -from pathlib import Path -from typing import List - - -def get_dylib_paths(base_path: Path) -> List[Path]: - """base_path以下の全てのサブディレクトリにあるdylibファイルのリストを返す""" - return list(base_path.glob("**/*.dylib")) - - -def get_rpaths(shared_lib_path: Path) -> List[Path]: - """引数で指定された共有ライブラリのrpathのリストを返す""" - proc = subprocess.run(["otool", "-L", str(shared_lib_path)], stdout=subprocess.PIPE) - output = proc.stdout.decode("utf-8") - paths = [ - Path(line.lstrip().split(" ", maxsplit=1)[0]) - for line in output.splitlines()[1:] - ] - # 得られたパスのリストのうち、共有ライブラリ自体とライブラリ名が同じものは - # rpath ではなく install ID というものなので除外 - return [ - path - for path in paths - if path.name.split(".")[0] != shared_lib_path.name.split(".")[0] - ] - - -def is_distributable_rpath(rpath: Path) -> bool: - """開発環境にインストールされたパッケージに依存しないrpathかどうか""" - # 以下のプレフィックスで始まるrpathは配布に際して問題がない - # - プレースホルダ。実行時に自動で解決される - # - @executable_path/ - # - @loader_path/ - # - @rpath/ - # - システム標準のライブラリがあるディレクトリ - # - /usr/lib/ - # - /System/Library/Frameworks/ - # - /System/Library/PrivateFrameworks/ - DISTRIBUTABLE_PREFIXES = [ - "@executable_path/", - "@loader_path/", - "@rpath/", - "/usr/lib/", - "/System/Library/Frameworks/", - "/System/Library/PrivateFrameworks/", - ] - result = False - - for prefix in DISTRIBUTABLE_PREFIXES: - if str(rpath).startswith(prefix): - result = True - break - else: - continue - - return result - - -def change_rpath(old_rpath: Path, new_rpath: Path, dylib_path: Path, base_path: Path): - """dylib_pathで指定されたdylibのrpathを、old_rpathから、new_rpath（base_pathからの相対パスに変換したもの）に変更する""" - relative_new_rpath = new_rpath.relative_to(base_path) - subprocess.run( - [ - "install_name_tool", - "-change", - old_rpath, - "@rpath/" + str(relative_new_rpath), - dylib_path, - ] - ) - - -class SharedLib: - """共有ライブラリの情報""" - - __path: Path - __rpaths: List[Path] - - def __init__(self, shared_lib_path: Path): - self.__path = shared_lib_path - self.__rpaths = get_rpaths(shared_lib_path) - - @property - def path(self) -> Path: - return self.__path - - def get_non_distributable_rpaths(self) -> List[Path]: - """rpathのうち、開発環境に依存しているもののリスト""" - return [rpath for rpath in self.__rpaths if not is_distributable_rpath(rpath)] diff --git a/build_util/macos/copy_missing_dylibs.py b/build_util/macos/copy_missing_dylibs.py deleted file mode 100644 index cd1a81fb0..000000000 --- a/build_util/macos/copy_missing_dylibs.py +++ /dev/null @@ -1,54 +0,0 @@ -""" -配布物内の.dylibファイルの不足を解消するためのスクリプト - -引数で指定したbase_directory以下にある.dylibファイルのrpathをチェックし、 -rpathの指す.dylibファイルがbase_directory以下に存在しなかった場合、 -rpathの指している場所からその.dylibファイルをbase_directory直下へとコピーする。 -""" - -import argparse -import shutil -import sys -from pathlib import Path -from typing import List, Set - -from build_util_macos.shlib_tools import SharedLib, get_dylib_paths - -parser = argparse.ArgumentParser() -parser.add_argument( - "base_directory", help="copy the missing dylibs under base_directory", type=str -) -args = parser.parse_args() -base_dir_path = Path(args.base_directory) - -if not (base_dir_path.exists() and base_dir_path.is_dir()): - print("could not find the directory:", str(base_dir_path), file=sys.stderr) - exit(1) - -# base_dir_path以下の全てのサブディレクトリを探索して得たdylibのリスト -dylib_paths: List[Path] = get_dylib_paths(base_dir_path) -# 全てのdylibのファイル名のリスト -dylib_names: List[str] = [path.name for path in dylib_paths] - -# 開発環境に依存したrpathを持つdylibのリスト -non_distributable_dylibs: List[SharedLib] = [] -for dylib_path in dylib_paths: - lib = SharedLib(dylib_path) - if lib.get_non_distributable_rpaths(): - non_distributable_dylibs.append(lib) - -# 開発環境に依存したrpathの集合 -non_distributable_rpaths: Set[Path] = set() -for dylib in non_distributable_dylibs: - rpaths: Set[Path] = set([rpath for rpath in dylib.get_non_distributable_rpaths()]) - non_distributable_rpaths = non_distributable_rpaths.union(rpaths) - -# rpathが指しているdylibのうち、base_dir_path以下に存在しないもののリスト -external_dylib_paths: List[Path] = [] -for rpath in non_distributable_rpaths: - if not (rpath.name in dylib_names): - external_dylib_paths.append(rpath) - -# 不足しているdylibをbase_dir_path直下にコピー -for dylib_path in external_dylib_paths: - shutil.copy(dylib_path, base_dir_path, follow_symlinks=True) diff --git a/build_util/macos/fix_rpaths.py b/build_util/macos/fix_rpaths.py deleted file mode 100644 index c2bd0f459..000000000 --- a/build_util/macos/fix_rpaths.py +++ /dev/null @@ -1,67 +0,0 @@ -""" -配布物内の.dylibファイルのrpathをどのようなユーザー環境においても有効になるように修正するスクリプト - -引数で指定したbase_directory以下にある.dylibファイルのrpathをチェックし、 -開発環境に依存した（配布先の環境に存在することが保証されていない）rpathであった場合、 -base_directory以下の.dylibファイルを相対パスで指すように変更する。 -（base_directory以下の.dylibファイルに不足がないことを前提とする。） -""" - -import argparse -import sys -from pathlib import Path -from typing import List, Set - -from build_util_macos.shlib_tools import SharedLib, change_rpath, get_dylib_paths - -parser = argparse.ArgumentParser() -parser.add_argument( - "base_directory", help="fix the rpaths of the dylibs under base_directory", type=str -) -args = parser.parse_args() -base_dir_path = Path(args.base_directory) - -if not (base_dir_path.exists() and base_dir_path.is_dir()): - print("could not find the directory:", str(base_dir_path), file=sys.stderr) - exit(1) - -# base_dir_path以下の全てのサブディレクトリを探索して得たdylibのリスト -internal_dylib_paths: List[Path] = get_dylib_paths(base_dir_path) -# 全てのdylibのファイル名のリスト -internal_dylib_names: List[str] = [path.name for path in internal_dylib_paths] - -# 開発環境に依存したrpathを持つdylibのリスト -non_distributable_dylibs: List[SharedLib] = [] -for internal_dylib_path in internal_dylib_paths: - lib = SharedLib(internal_dylib_path) - if lib.get_non_distributable_rpaths(): - non_distributable_dylibs.append(lib) - -# 開発環境に依存したrpathの集合 -non_distributable_rpaths: Set[Path] = set() -for dylib in non_distributable_dylibs: - rpaths: Set[Path] = set([rpath for rpath in dylib.get_non_distributable_rpaths()]) - non_distributable_rpaths = non_distributable_rpaths.union(rpaths) - -# rpathが指しているdylibのうち、base_dir_path以下に存在しないもののリスト -external_dylib_paths: List[Path] = [] -for rpath in non_distributable_rpaths: - if not (rpath.name in internal_dylib_names): - external_dylib_paths.append(rpath) - -# base_dir_path以下でdylibが不足している場合は、不足しているdylibを表示して終了 -if external_dylib_paths: - print( - f"following dylibs not found under base_dir_path ({str(base_dir_path)}):", - file=sys.stderr, - ) - for path in external_dylib_paths: - print(f"\t{path.name}", file=sys.stderr) - exit(1) - -# 開発環境に依存したrpathを、base_dir_path以下のdylibを指すように変更 -for dylib in non_distributable_dylibs: - for rpath in dylib.get_non_distributable_rpaths(): - for internal_dylib_path in internal_dylib_paths: - if internal_dylib_path.name == rpath.name: - change_rpath(rpath, internal_dylib_path, dylib.path, base_dir_path) diff --git a/run.spec b/run.spec new file mode 100644 index 000000000..03a8b313f --- /dev/null +++ b/run.spec @@ -0,0 +1,88 @@ +# -*- mode: python ; coding: utf-8 -*- +# このファイルはPyInstallerによって自動生成されたもので、それをカスタマイズして使用しています。 +from PyInstaller.utils.hooks import collect_data_files +import os + +datas = collect_data_files('pyopenjtalk') + + +block_cipher = None + + +a = Analysis( + ['run.py'], + pathex=[], + binaries=[], + datas=datas, + hiddenimports=[], + hookspath=[], + hooksconfig={}, + runtime_hooks=[], + excludes=[], + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher, + noarchive=False, +) + +pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher) + +exe = EXE( + pyz, + a.scripts, + a.binaries, + a.zipfiles, + a.datas, + [], + name='run', + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + upx_exclude=[], + runtime_tmpdir=None, + console=True, + disable_windowed_traceback=False, + argv_emulation=False, + target_arch=None, + codesign_identity=None, + entitlements_file=None, +) + +datas = [ + ('engine_manifest_assets', 'engine_manifest_assets', 'DATA'), + ('speaker_info', 'speaker_info', 'DATA'), + ('default.csv', 'default.csv', 'DATA'), + ('licenses.json', 'licenses.json', 'DATA'), + ('presets.yaml', 'presets.yaml', 'DATA'), +] + +# コアとONNX Runtimeはバイナリであるが、`binaries`に加えると +# 依存関係のパスがPyInstallerに書き換えらるので、`datas`に加える +# 参考: https://github.com/VOICEVOX/voicevox_engine/pull/446#issuecomment-1210052318 +libcore_path = os.environ.get('LIBCORE_PATH') +if libcore_path: + print('LIBCORE_PATH is found:', libcore_path) + if not os.path.isfile(libcore_path): + raise Exception("LIBCORE_PATH was found, but it is not file!") + filename = os.path.basename(libcore_path) + datas += [(filename, libcore_path, 'DATA')] + +libonnxruntime_path = os.environ.get('LIBONNXRUNTIME_PATH') +if libonnxruntime_path: + print('LIBONNXRUNTIME_PATH is found:', libonnxruntime_path) + if not os.path.isfile(libonnxruntime_path): + raise Exception("LIBCORE_PATH was found, but it is not file!") + filename = os.path.basename(libonnxruntime_path) + datas += [(filename, libonnxruntime_path, 'DATA')] + +coll = COLLECT( + exe, + [], + [], + datas, + strip=False, + upx=True, + upx_exclude=[], + name='run.dist', +) From 54357d4220aa52cafcdc556e2dd4e82370846355 Mon Sep 17 00:00:00 2001 From: Yuto Ashida Date: Mon, 15 Aug 2022 12:23:18 +0900 Subject: [PATCH 3/9] =?UTF-8?q?Linux=E3=83=93=E3=83=AB=E3=83=89=E3=82=92Py?= =?UTF-8?q?Installer=E3=81=B8=E7=A7=BB=E8=A1=8C=E3=81=97=E3=80=81=E4=BB=96?= =?UTF-8?q?OS=E3=81=A8=E3=83=93=E3=83=AB=E3=83=89=E3=82=B8=E3=83=A7?= =?UTF-8?q?=E3=83=96=E3=82=92=E7=B5=B1=E5=90=88=E3=81=99=E3=82=8B=20=20(#4?= =?UTF-8?q?50)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * use pyinstaller in github action for linux build and integrated job * fix nvidia gpg key * add build essential setup step * update ubuntu version * fix miss * fix error * fix download core * fix shell miss * fix typo * fix miss * fix tar command * downgrade ubuntu version and fix pip command * add create symlink step * fix miss * update cuda toolkit and remove prepare cuda apt key step * command integrating * use dist/run.dist directory in windows * use setup python4 and remove no-cache dir option --- .github/workflows/build.yml | 283 +++++++++++++----------------------- 1 file changed, 104 insertions(+), 179 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c3dd7f71f..23a19d874 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,136 +29,7 @@ env: ${{ github.event.release.tag_name || github.event.inputs.version || 'latest' }} jobs: - # Build Linux binary (push only buildcache image) - build-linux: - strategy: - matrix: - os: [ubuntu-latest] - tag: - - build-cpu-ubuntu18.04 - - build-nvidia-ubuntu18.04 - include: - - tag: build-cpu-ubuntu18.04 - runtime_tag: cpu-ubuntu18.04 # for cache use - target: build-env - base_image: ubuntu:bionic - base_runtime_image: ubuntu:bionic - voicevox_core_asset_prefix: voicevox_core-linux-x64-cpu - onnxruntime_url: https://github.com/microsoft/onnxruntime/releases/download/v1.10.0/onnxruntime-linux-x64-1.10.0.tgz - artifact_name: linux-cpu - nuitka_cache_path: nuitka_cache - - tag: build-nvidia-ubuntu18.04 - runtime_tag: nvidia-ubuntu18.04 # for cache use - target: build-env - base_image: ubuntu:bionic - base_runtime_image: nvidia/cuda:11.4.2-cudnn8-runtime-ubuntu18.04 - voicevox_core_asset_prefix: voicevox_core-linux-x64-gpu - onnxruntime_url: https://github.com/microsoft/onnxruntime/releases/download/v1.10.0/onnxruntime-linux-x64-gpu-1.10.0.tgz - artifact_name: linux-nvidia - nuitka_cache_path: nuitka_cache - - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v2 - - - name: Setup Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v1 - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - # Download VOICEVOX RESOURCE - - name: Prepare VOICEVOX RESOURCE cache - uses: actions/cache@v2 - id: voicevox-resource-cache - with: - key: voicevox-resource-${{ env.VOICEVOX_RESOURCE_VERSION }} - path: download/resource - - - name: Checkout VOICEVOX RESOURCE - if: steps.voicevox-resource-cache.outputs.cache-hit != 'true' - uses: actions/checkout@v2 - with: - repository: VOICEVOX/voicevox_resource - ref: ${{ env.VOICEVOX_RESOURCE_VERSION }} - path: download/resource - - # Merge VOICEVOX RESOURCE - - name: Merge VOICEVOX RESOURCE - shell: bash - env: - DOWNLOAD_RESOURCE_PATH: download/resource - run: bash build_util/merge_voicevox_resource.bash - - # NOTE: `load: true` may silently fail when the GitHub Actions disk (14GB) is full. - # https://docs.github.com/ja/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources - - name: Create binary build environment with Docker - uses: docker/build-push-action@v2 - env: - IMAGE_TAG: ${{ env.IMAGE_NAME }}:${{ matrix.tag }}${{ (matrix.tag != '' && '-') || '' }}latest - RUNTIME_IMAGE_TAG: ${{ env.IMAGE_NAME }}:${{ matrix.runtime_tag }}${{ (matrix.runtime_tag != '' && '-') || '' }}latest - VOICEVOX_CORE_ASSET_NAME: ${{ matrix.voicevox_core_asset_prefix }}-${{ env.VOICEVOX_CORE_VERSION }} - with: - context: . - builder: ${{ steps.buildx.outputs.name }} - file: ./Dockerfile - build-args: | - BASE_IMAGE=${{ matrix.base_image }} - BASE_RUNTIME_IMAGE=${{ matrix.base_runtime_image }} - PYTHON_VERSION=${{ env.PYTHON_VERSION }} - VOICEVOX_ENGINE_VERSION=${{ env.VOICEVOX_ENGINE_VERSION }} - VOICEVOX_CORE_ASSET_NAME=${{ env.VOICEVOX_CORE_ASSET_NAME }} - VOICEVOX_CORE_VERSION=${{ env.VOICEVOX_CORE_VERSION }} - VOICEVOX_RESOURCE_VERSION=${{ env.VOICEVOX_RESOURCE_VERSION }} - ONNXRUNTIME_URL=${{ matrix.onnxruntime_url }} - target: ${{ matrix.target }} - load: true - tags: | - ${{ env.IMAGE_TAG }} - cache-from: | - type=registry,ref=${{ env.IMAGE_TAG }}-buildcache - type=registry,ref=${{ env.RUNTIME_IMAGE_TAG }}-buildcache - cache-to: type=registry,ref=${{ env.IMAGE_TAG }}-buildcache,mode=max - - # Build run.py with Nuitka in Docker - - name: Cache Nuitka (ccache, module-cache) - uses: actions/cache@v2 - id: nuitka-cache - with: - path: ${{ matrix.nuitka_cache_path }} - key: ${{ runner.os }}-nuitka-${{ matrix.tag }}-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-nuitka-${{ matrix.tag }}- - - - name: Build run.py with Nuitka in Docker - env: - IMAGE_TAG: ${{ env.IMAGE_NAME }}:${{ matrix.tag }}${{ (matrix.tag != '' && '-') || '' }}latest - run: | - docker run --rm \ - -v "$(readlink -f "${{ matrix.nuitka_cache_path }}"):/home/user/.cache/Nuitka" \ - -v "$(readlink -f "build"):/opt/voicevox_engine_build" \ - "${{ env.IMAGE_TAG }}" - - # FIXME: versioned name may be useful; but - # actions/download-artifact and dawidd6/download-artifact do not support - # wildcard / forward-matching yet. - # Currently, It is good to use static artifact name for future binary test workflow. - # https://github.com/actions/toolkit/blob/ea81280a4d48fb0308d40f8f12ae00d117f8acb9/packages/artifact/src/internal/artifact-client.ts#L147 - # https://github.com/dawidd6/action-download-artifact/blob/af92a8455a59214b7b932932f2662fdefbd78126/main.js#L113 - - uses: actions/upload-artifact@v2 - # env: - # VERSIONED_ARTIFACT_NAME: | - # ${{ format('{0}-{1}', matrix.artifact_name, (env.VOICEVOX_ENGINE_VERSION != 'latest' && env.VOICEVOX_ENGINE_VERSION) || github.sha) }} - with: - name: ${{ matrix.artifact_name }} - path: build/run.dist/ - - build-windows-and-mac: + build-all: environment: ${{ github.event.inputs.code_signing == 'true' && 'code_signing' }} # コード署名用のenvironment strategy: matrix: @@ -190,6 +61,20 @@ jobs: voicevox_core_asset_prefix: voicevox_core-osx-universal2-cpu onnxruntime_url: https://github.com/microsoft/onnxruntime/releases/download/v1.10.0/onnxruntime-osx-universal2-1.10.0.tgz artifact_name: macos-x64 + # Linux CPU + - os: ubuntu-18.04 + architecture: "x64" + voicevox_core_asset_prefix: voicevox_core-linux-x64-cpu + onnxruntime_url: https://github.com/microsoft/onnxruntime/releases/download/v1.10.0/onnxruntime-linux-x64-1.10.0.tgz + artifact_name: linux-cpu + # Linux NVIDIA GPU + - os: ubuntu-18.04 + architecture: "x64" + voicevox_core_asset_prefix: voicevox_core-linux-x64-gpu + onnxruntime_url: https://github.com/microsoft/onnxruntime/releases/download/v1.10.0/onnxruntime-linux-x64-gpu-1.10.0.tgz + cuda_version: "11.4.2" + cudnn_url: https://developer.download.nvidia.com/compute/redist/cudnn/v8.2.4/cudnn-11.4-linux-x64-v8.2.4.15.tgz + artifact_name: linux-nvidia runs-on: ${{ matrix.os }} @@ -221,13 +106,13 @@ jobs: - name: Setup CUDA if: matrix.cuda_version != '' && steps.cuda-dll-cache.outputs.cache-hit != 'true' - uses: Jimver/cuda-toolkit@v0.2.5 + uses: Jimver/cuda-toolkit@v0.2.8 id: cuda-toolkit with: method: network cuda: ${{ matrix.cuda_version }} - - name: Extract CUDA DLL + - name: Extract CUDA Dynamic Libraries if: matrix.cuda_version != '' && steps.cuda-dll-cache.outputs.cache-hit != 'true' shell: bash run: | @@ -236,9 +121,24 @@ jobs: CUDA_ROOT=$( echo "${{ steps.cuda-toolkit.outputs.CUDA_PATH }}" | tr '\\' '/' ) mkdir -p download/cuda/bin - mv "${CUDA_ROOT}/bin/"*.dll download/cuda/bin/ + if [[ ${{ matrix.os }} == windows-* ]]; then + mv "${CUDA_ROOT}/bin/"*.dll download/cuda/bin/ - rm -rf "${CUDA_ROOT}" + rm -rf "${CUDA_ROOT}" + else + cp "${CUDA_ROOT}/lib64/"libcublas.so.* download/cuda/bin/ + cp "${CUDA_ROOT}/lib64/"libcublasLt.so.* download/cuda/bin/ + cp "${CUDA_ROOT}/lib64/"libcudart.so.* download/cuda/bin/ + cp "${CUDA_ROOT}/lib64/"libcufft.so.* download/cuda/bin/ + cp "${CUDA_ROOT}/lib64/"libcurand.so.* download/cuda/bin/ + + # remove unneed full version libraries + rm -f download/cuda/bin/libcublas.so.*.* + rm -f download/cuda/bin/libcublasLt.so.*.* + rm -f download/cuda/bin/libcufft.so.*.* + rm -f download/cuda/bin/libcurand.so.*.* + rm -f download/cuda/bin/libcudart.so.*.*.* + fi - name: Show disk space (debug info) if: matrix.cuda_version != '' @@ -261,21 +161,37 @@ jobs: key: ${{ matrix.os }}-cudnn-dll-${{ hashFiles('download/cudnn_url.txt') }}-v1 path: download/cudnn - - name: Download and extract cuDNN DLLs + - name: Download and extract cuDNN Dynamic Libraries if: matrix.cudnn_url != '' && steps.cudnn-dll-cache.outputs.cache-hit != 'true' shell: bash run: | set -eux - curl -L "${{ matrix.cudnn_url }}" > download/cudnn.zip + if [[ ${{ matrix.os }} == windows-* ]]; then + curl -L "${{ matrix.cudnn_url }}" > download/cudnn.zip + + unzip download/cudnn.zip cuda/bin/*.dll -d download/cudnn_tmp + + mkdir -p download/cudnn/bin + mv download/cudnn_tmp/cuda/bin/*.dll download/cudnn/bin/ + rm -rf download/cudnn_tmp + + rm download/cudnn.zip + else + curl -L "${{ matrix.cudnn_url }}" > download/cudnn.tgz - unzip download/cudnn.zip cuda/bin/*.dll -d download/cudnn_tmp + tar -zxf download/cudnn.tgz -C download/ + + mkdir -p download/cudnn/bin + cp download/cuda/lib64/libcudnn.so.* download/cudnn/bin/ + cp download/cuda/lib64/libcudnn_*_infer.so.* download/cudnn/bin/ - mkdir -p download/cudnn/bin - mv download/cudnn_tmp/cuda/bin/*.dll download/cudnn/bin/ - rm -rf download/cudnn_tmp + # remove unneed full version libraries + rm -f download/cudnn/bin/libcudnn.so.*.* + rm -f download/cudnn/bin/libcudnn_*_infer.so.*.* - rm download/cudnn.zip + rm download/cudnn.tgz + fi - name: Show disk space (debug info) if: matrix.cudnn_url != '' @@ -290,7 +206,7 @@ jobs: # Python install path of windows: C:/hostedtoolcache/windows/Python - name: Setup Python id: setup-python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ env.PYTHON_VERSION }} architecture: ${{ matrix.architecture }} @@ -380,15 +296,13 @@ jobs: rm download/onnxruntime.zip - - name: Download ONNX Runtime (Mac) - if: steps.onnxruntime-cache.outputs.cache-hit != 'true' && startsWith(matrix.os, 'macos-') + - name: Download ONNX Runtime (Mac/Linux) + if: steps.onnxruntime-cache.outputs.cache-hit != 'true' && startsWith(matrix.os, 'windows-') != true shell: bash run: | curl -L "${{ matrix.onnxruntime_url }}" > download/onnxruntime.tgz - - # extract only dylibs mkdir -p download/onnxruntime - tar xf "download/onnxruntime.tgz" -C "download/onnxruntime" --strip-components 1 'onnxruntime-*/lib/*.dylib' + tar xf "download/onnxruntime.tgz" -C "download/onnxruntime" --strip-components 1 rm download/onnxruntime.tgz - name: Show disk space (debug info) @@ -441,8 +355,10 @@ jobs: # unzip download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip -d download/ if [[ ${{ matrix.os }} == windows-* ]]; then 7z x -o"download" download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip - else + elif [[ ${{ matrix.os }} == mac-* ]]; then ditto -x -k --sequesterRsrc --rsrc download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip download/ + else + unzip download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip -d download/ fi mkdir -p download/core mv download/${{ env.VOICEVOX_CORE_ASSET_NAME }}/* download/core @@ -473,8 +389,13 @@ jobs: LIBONNXRUNTIME_PATH=download/onnxruntime/lib/libonnxruntime.dylib else sed -i "s/__version__ = \"latest\"/__version__ = \"${{ env.VOICEVOX_ENGINE_VERSION }}\"/" voicevox_engine/__init__.py - LIBCORE_PATH=download/core/core.dll - LIBONNXRUNTIME_PATH=download/onnxruntime/lib/onnxruntime.dll + if [[ ${{ matrix.os }} == windows-* ]]; then + LIBCORE_PATH=download/core/core.dll + LIBONNXRUNTIME_PATH=download/onnxruntime/lib/onnxruntime.dll + else + LIBCORE_PATH=download/core/libcore.so + LIBONNXRUNTIME_PATH=download/onnxruntime/lib/libonnxruntime.so + fi fi LIBCORE_PATH="$LIBCORE_PATH" \ @@ -497,35 +418,51 @@ jobs: run: | set -eux - # Build artifact directory - mkdir -p artifact - ln -sf "$(pwd)/dist/run.dist"/* artifact/ - # Copy DLL dependencies if [ -f "download/onnxruntime/lib/onnxruntime_providers_cuda.dll" ]; then # ONNX Runtime providers (PyInstaller does not copy dynamic loaded libraries) - ln -sf "$(pwd)/download/onnxruntime/lib"/onnxruntime_*.dll artifact/ + ln -sf "$(pwd)/download/onnxruntime/lib"/onnxruntime_*.dll dist/run.dist/ # CUDA - ln -sf "$(pwd)/download/cuda/bin"/cublas64_*.dll artifact/ - ln -sf "$(pwd)/download/cuda/bin"/cublasLt64_*.dll artifact/ - ln -sf "$(pwd)/download/cuda/bin"/cudart64_*.dll artifact/ - ln -sf "$(pwd)/download/cuda/bin"/cufft64_*.dll artifact/ - ln -sf "$(pwd)/download/cuda/bin"/curand64_*.dll artifact/ + ln -sf "$(pwd)/download/cuda/bin"/cublas64_*.dll dist/run.dist/ + ln -sf "$(pwd)/download/cuda/bin"/cublasLt64_*.dll dist/run.dist/ + ln -sf "$(pwd)/download/cuda/bin"/cudart64_*.dll dist/run.dist/ + ln -sf "$(pwd)/download/cuda/bin"/cufft64_*.dll dist/run.dist/ + ln -sf "$(pwd)/download/cuda/bin"/curand64_*.dll dist/run.dist/ # cuDNN - ln -sf "$(pwd)/download/cudnn/bin"/cudnn64_*.dll artifact/ - ln -sf "$(pwd)/download/cudnn/bin"/cudnn_*_infer64*.dll artifact/ + ln -sf "$(pwd)/download/cudnn/bin"/cudnn64_*.dll dist/run.dist/ + ln -sf "$(pwd)/download/cudnn/bin"/cudnn_*_infer64*.dll dist/run.dist/ fi if [[ ${{ matrix.artifact_name }} == *-directml ]]; then # DirectML - ln -sf "$(pwd)/download/directml"/DirectML.dll artifact/ + ln -sf "$(pwd)/download/directml"/DirectML.dll dist/run.dist/ fi # pysoundfile - ln -sf "${{ env.PYTHON_SITE_PACKAGES_DIR }}/_soundfile_data" artifact/ + ln -sf "${{ env.PYTHON_SITE_PACKAGES_DIR }}/_soundfile_data" dist/run.dist/ + + - name: Create symlink of CUDA dependencies + if: startsWith(matrix.os, 'ubuntu-') && endsWith(matrix.artifact_name, 'nvidia') + shell: bash + run: | + set -eux + + # ONNX Runtime providers (PyInstaller does not copy dynamic loaded libraries) + ln -sf "$(pwd)/download/onnxruntime/lib"/libonnxruntime_*.so dist/run.dist/ + + # CUDA + ln -sf "$(pwd)/download/cuda/bin"/libcublas.so.* dist/run.dist/ + ln -sf "$(pwd)/download/cuda/bin"/libcublasLt.so.* dist/run.dist/ + ln -sf "$(pwd)/download/cuda/bin"/libcudart.so.* dist/run.dist/ + ln -sf "$(pwd)/download/cuda/bin"/libcufft.so.* dist/run.dist/ + ln -sf "$(pwd)/download/cuda/bin"/libcurand.so.* dist/run.dist/ + + # cuDNN + ln -sf "$(pwd)/download/cudnn/bin"/libcudnn.so.* dist/run.dist/ + ln -sf "$(pwd)/download/cudnn/bin"/libcudnn_*_infer.so.* dist/run.dist/ - name: Set @rpath to @executable_path if: startsWith(matrix.os, 'macos-') @@ -536,7 +473,7 @@ jobs: if: github.event.inputs.code_signing == 'true' && startsWith(matrix.os, 'windows-') shell: bash run: | - bash build_util/codesign.bash "artifact/run.exe" + bash build_util/codesign.bash "dist/run.dist/run.exe" env: CERT_BASE64: ${{ secrets.CERT_BASE64 }} CERT_PASSWORD: ${{ secrets.CERT_PASSWORD }} @@ -547,19 +484,7 @@ jobs: # Currently, It is good to use static artifact name for future binary test workflow. # https://github.com/actions/toolkit/blob/ea81280a4d48fb0308d40f8f12ae00d117f8acb9/packages/artifact/src/internal/artifact-client.ts#L147 # https://github.com/dawidd6/action-download-artifact/blob/af92a8455a59214b7b932932f2662fdefbd78126/main.js#L113 - - name: Upload artifact (Windows) - if: startsWith(matrix.os, 'windows-') - uses: actions/upload-artifact@v2 - # env: - # VERSIONED_ARTIFACT_NAME: | - # ${{ format('{0}-{1}', matrix.artifact_name, (env.VOICEVOX_ENGINE_VERSION != 'latest' && env.VOICEVOX_ENGINE_VERSION) || github.sha) }} - with: - name: ${{ matrix.artifact_name }} - path: | - artifact/ - - - name: Upload artifact (Mac) - if: startsWith(matrix.os, 'macos-') + - name: Upload artifact uses: actions/upload-artifact@v2 # env: # VERSIONED_ARTIFACT_NAME: | @@ -570,7 +495,7 @@ jobs: upload-to-release: if: (github.event.release.tag_name || github.event.inputs.version) != '' - needs: [build-linux, build-windows-and-mac] + needs: [build-all] runs-on: ubuntu-latest strategy: matrix: From a3deaa7d7114d289341b926d2b906560bd161efe Mon Sep 17 00:00:00 2001 From: Yuto Ashida Date: Wed, 21 Sep 2022 22:40:11 +0900 Subject: [PATCH 4/9] =?UTF-8?q?[PyInstaller]=20Linux=E3=83=93=E3=83=AB?= =?UTF-8?q?=E3=83=89=E3=81=8C=E3=81=86=E3=81=BE=E3=81=8F=E5=8B=95=E3=81=8B?= =?UTF-8?q?=E3=81=AA=E3=81=84=E5=95=8F=E9=A1=8C=E3=82=92=E4=BF=AE=E6=AD=A3?= =?UTF-8?q?=20(#468)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix faild start up when using gpu --- .github/workflows/build.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 23a19d874..b4df1559c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -94,6 +94,13 @@ jobs: shell: bash run: brew install gnu-sed + # ONNX Runtime providersとCUDA周りをリンクするために使う + - name: Install patchelf + if: startsWith(matrix.os, 'ubuntu-') && endsWith(matrix.artifact_name, 'nvidia') + run: | + sudo apt-get update + sudo apt-get install -y patchelf + # Download CUDA - name: Prepare CUDA DLL cache if: matrix.cuda_version != '' @@ -451,6 +458,7 @@ jobs: set -eux # ONNX Runtime providers (PyInstaller does not copy dynamic loaded libraries) + patchelf --set-rpath '$ORIGIN' "$(pwd)/download/onnxruntime/lib"/libonnxruntime_providers_*.so ln -sf "$(pwd)/download/onnxruntime/lib"/libonnxruntime_*.so dist/run.dist/ # CUDA From 6ad80c3ea15e05792c38ff9ad5bc81829ebff467 Mon Sep 17 00:00:00 2001 From: Yuto Ashida Date: Fri, 4 Nov 2022 05:48:32 +0900 Subject: [PATCH 5/9] =?UTF-8?q?PyInstaller:=20=E3=82=A8=E3=83=B3=E3=82=B8?= =?UTF-8?q?=E3=83=B3=E4=BD=BF=E7=94=A8=E6=99=82=E3=81=AB=E4=B8=80=E6=99=82?= =?UTF-8?q?=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E3=81=8C=E6=AE=8B=E3=81=A3?= =?UTF-8?q?=E3=81=A6=E3=81=97=E3=81=BE=E3=81=86=E5=95=8F=E9=A1=8C=E3=81=AE?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3=20(#503)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build.yml | 43 +++++++++++------------- run.spec | 67 +++++++++++++++++-------------------- 2 files changed, 50 insertions(+), 60 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b4df1559c..1b017d8fa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -429,28 +429,25 @@ jobs: if [ -f "download/onnxruntime/lib/onnxruntime_providers_cuda.dll" ]; then # ONNX Runtime providers (PyInstaller does not copy dynamic loaded libraries) - ln -sf "$(pwd)/download/onnxruntime/lib"/onnxruntime_*.dll dist/run.dist/ + ln -sf "$(pwd)/download/onnxruntime/lib"/onnxruntime_*.dll dist/run/ # CUDA - ln -sf "$(pwd)/download/cuda/bin"/cublas64_*.dll dist/run.dist/ - ln -sf "$(pwd)/download/cuda/bin"/cublasLt64_*.dll dist/run.dist/ - ln -sf "$(pwd)/download/cuda/bin"/cudart64_*.dll dist/run.dist/ - ln -sf "$(pwd)/download/cuda/bin"/cufft64_*.dll dist/run.dist/ - ln -sf "$(pwd)/download/cuda/bin"/curand64_*.dll dist/run.dist/ + ln -sf "$(pwd)/download/cuda/bin"/cublas64_*.dll dist/run/ + ln -sf "$(pwd)/download/cuda/bin"/cublasLt64_*.dll dist/run/ + ln -sf "$(pwd)/download/cuda/bin"/cudart64_*.dll dist/run/ + ln -sf "$(pwd)/download/cuda/bin"/cufft64_*.dll dist/run/ + ln -sf "$(pwd)/download/cuda/bin"/curand64_*.dll dist/run/ # cuDNN - ln -sf "$(pwd)/download/cudnn/bin"/cudnn64_*.dll dist/run.dist/ - ln -sf "$(pwd)/download/cudnn/bin"/cudnn_*_infer64*.dll dist/run.dist/ + ln -sf "$(pwd)/download/cudnn/bin"/cudnn64_*.dll dist/run/ + ln -sf "$(pwd)/download/cudnn/bin"/cudnn_*_infer64*.dll dist/run/ fi if [[ ${{ matrix.artifact_name }} == *-directml ]]; then # DirectML - ln -sf "$(pwd)/download/directml"/DirectML.dll dist/run.dist/ + ln -sf "$(pwd)/download/directml"/DirectML.dll dist/run/ fi - # pysoundfile - ln -sf "${{ env.PYTHON_SITE_PACKAGES_DIR }}/_soundfile_data" dist/run.dist/ - - name: Create symlink of CUDA dependencies if: startsWith(matrix.os, 'ubuntu-') && endsWith(matrix.artifact_name, 'nvidia') shell: bash @@ -459,29 +456,29 @@ jobs: # ONNX Runtime providers (PyInstaller does not copy dynamic loaded libraries) patchelf --set-rpath '$ORIGIN' "$(pwd)/download/onnxruntime/lib"/libonnxruntime_providers_*.so - ln -sf "$(pwd)/download/onnxruntime/lib"/libonnxruntime_*.so dist/run.dist/ + ln -sf "$(pwd)/download/onnxruntime/lib"/libonnxruntime_*.so dist/run/ # CUDA - ln -sf "$(pwd)/download/cuda/bin"/libcublas.so.* dist/run.dist/ - ln -sf "$(pwd)/download/cuda/bin"/libcublasLt.so.* dist/run.dist/ - ln -sf "$(pwd)/download/cuda/bin"/libcudart.so.* dist/run.dist/ - ln -sf "$(pwd)/download/cuda/bin"/libcufft.so.* dist/run.dist/ - ln -sf "$(pwd)/download/cuda/bin"/libcurand.so.* dist/run.dist/ + ln -sf "$(pwd)/download/cuda/bin"/libcublas.so.* dist/run/ + ln -sf "$(pwd)/download/cuda/bin"/libcublasLt.so.* dist/run/ + ln -sf "$(pwd)/download/cuda/bin"/libcudart.so.* dist/run/ + ln -sf "$(pwd)/download/cuda/bin"/libcufft.so.* dist/run/ + ln -sf "$(pwd)/download/cuda/bin"/libcurand.so.* dist/run/ # cuDNN - ln -sf "$(pwd)/download/cudnn/bin"/libcudnn.so.* dist/run.dist/ - ln -sf "$(pwd)/download/cudnn/bin"/libcudnn_*_infer.so.* dist/run.dist/ + ln -sf "$(pwd)/download/cudnn/bin"/libcudnn.so.* dist/run/ + ln -sf "$(pwd)/download/cudnn/bin"/libcudnn_*_infer.so.* dist/run/ - name: Set @rpath to @executable_path if: startsWith(matrix.os, 'macos-') run: | - install_name_tool -add_rpath @executable_path/. dist/run.dist/run + install_name_tool -add_rpath @executable_path/. dist/run/run - name: Code signing if: github.event.inputs.code_signing == 'true' && startsWith(matrix.os, 'windows-') shell: bash run: | - bash build_util/codesign.bash "dist/run.dist/run.exe" + bash build_util/codesign.bash "dist/run/run.exe" env: CERT_BASE64: ${{ secrets.CERT_BASE64 }} CERT_PASSWORD: ${{ secrets.CERT_PASSWORD }} @@ -499,7 +496,7 @@ jobs: # ${{ format('{0}-{1}', matrix.artifact_name, (env.VOICEVOX_ENGINE_VERSION != 'latest' && env.VOICEVOX_ENGINE_VERSION) || github.sha) }} with: name: ${{ matrix.artifact_name }} - path: dist/run.dist/ + path: dist/run/ upload-to-release: if: (github.event.release.tag_name || github.event.inputs.version) != '' diff --git a/run.spec b/run.spec index 03a8b313f..f262338fb 100644 --- a/run.spec +++ b/run.spec @@ -3,7 +3,31 @@ from PyInstaller.utils.hooks import collect_data_files import os -datas = collect_data_files('pyopenjtalk') +datas = [ + ('engine_manifest_assets', 'engine_manifest_assets'), + ('speaker_info', 'speaker_info'), + ('default.csv', '.'), + ('licenses.json', '.'), + ('presets.yaml', '.'), +] +datas += collect_data_files('pyopenjtalk') + +# コアとONNX Runtimeはバイナリであるが、`binaries`に加えると +# 依存関係のパスがPyInstallerに書き換えらるので、`datas`に加える +# 参考: https://github.com/VOICEVOX/voicevox_engine/pull/446#issuecomment-1210052318 +libcore_path = os.environ.get('LIBCORE_PATH') +if libcore_path: + print('LIBCORE_PATH is found:', libcore_path) + if not os.path.isfile(libcore_path): + raise Exception("LIBCORE_PATH was found, but it is not file!") + datas += [(libcore_path, ".")] + +libonnxruntime_path = os.environ.get('LIBONNXRUNTIME_PATH') +if libonnxruntime_path: + print('LIBONNXRUNTIME_PATH is found:', libonnxruntime_path) + if not os.path.isfile(libonnxruntime_path): + raise Exception("LIBCORE_PATH was found, but it is not file!") + datas += [(libonnxruntime_path, ".")] block_cipher = None @@ -30,17 +54,13 @@ pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher) exe = EXE( pyz, a.scripts, - a.binaries, - a.zipfiles, - a.datas, [], + exclude_binaries=True, name='run', debug=False, bootloader_ignore_signals=False, strip=False, upx=True, - upx_exclude=[], - runtime_tmpdir=None, console=True, disable_windowed_traceback=False, argv_emulation=False, @@ -49,40 +69,13 @@ exe = EXE( entitlements_file=None, ) -datas = [ - ('engine_manifest_assets', 'engine_manifest_assets', 'DATA'), - ('speaker_info', 'speaker_info', 'DATA'), - ('default.csv', 'default.csv', 'DATA'), - ('licenses.json', 'licenses.json', 'DATA'), - ('presets.yaml', 'presets.yaml', 'DATA'), -] - -# コアとONNX Runtimeはバイナリであるが、`binaries`に加えると -# 依存関係のパスがPyInstallerに書き換えらるので、`datas`に加える -# 参考: https://github.com/VOICEVOX/voicevox_engine/pull/446#issuecomment-1210052318 -libcore_path = os.environ.get('LIBCORE_PATH') -if libcore_path: - print('LIBCORE_PATH is found:', libcore_path) - if not os.path.isfile(libcore_path): - raise Exception("LIBCORE_PATH was found, but it is not file!") - filename = os.path.basename(libcore_path) - datas += [(filename, libcore_path, 'DATA')] - -libonnxruntime_path = os.environ.get('LIBONNXRUNTIME_PATH') -if libonnxruntime_path: - print('LIBONNXRUNTIME_PATH is found:', libonnxruntime_path) - if not os.path.isfile(libonnxruntime_path): - raise Exception("LIBCORE_PATH was found, but it is not file!") - filename = os.path.basename(libonnxruntime_path) - datas += [(filename, libonnxruntime_path, 'DATA')] - coll = COLLECT( exe, - [], - [], - datas, + a.binaries, + a.zipfiles, + a.datas, strip=False, upx=True, upx_exclude=[], - name='run.dist', + name='run', ) From 49a6f6f5d589702ed5b86ceadc23344c56064d04 Mon Sep 17 00:00:00 2001 From: Yuto Ashida Date: Sun, 6 Nov 2022 10:33:00 +0900 Subject: [PATCH 6/9] =?UTF-8?q?PyInstaller:=20Dockerfile=E3=81=8B=E3=82=89?= =?UTF-8?q?=E3=83=93=E3=83=AB=E3=83=89=E9=96=A2=E9=80=A3=E3=81=AE=E3=82=B3?= =?UTF-8?q?=E3=83=BC=E3=83=89=E3=82=92=E5=89=8A=E9=99=A4=20(#484)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * remove build-env from docker file * remove nuitka build issue --- Dockerfile | 125 ----------------------------------------------------- 1 file changed, 125 deletions(-) diff --git a/Dockerfile b/Dockerfile index 793ef83de..d2f8e3522 100644 --- a/Dockerfile +++ b/Dockerfile @@ -185,9 +185,6 @@ COPY --from=compile-python-env /opt/python /opt/python ADD ./requirements.txt /tmp/ RUN <=60.7.0 - # https://github.com/Nuitka/Nuitka/issues/1406 - gosu user /opt/python/bin/python3 -m pip install --upgrade pip setuptools==60.6.0 wheel gosu user /opt/python/bin/pip3 install -r /tmp/requirements.txt EOF @@ -272,125 +269,3 @@ CMD [ "gosu", "user", "/opt/python/bin/python3", "./run.py", "--voicelib_dir", " # Enable use_gpu FROM runtime-env AS runtime-nvidia-env CMD [ "gosu", "user", "/opt/python/bin/python3", "./run.py", "--use_gpu", "--voicelib_dir", "/opt/voicevox_core/", "--runtime_dir", "/opt/onnxruntime/lib", "--host", "0.0.0.0" ] - -# Binary build environment (common to CPU, GPU) -FROM runtime-env AS build-env - -# Install ccache for Nuitka cache -# chrpath: required for nuitka build; 'RPATH' settings in used shared -RUN < /opt/voicevox_engine/engine_manifest_assets/dependency_licenses.json - # FIXME: VOICEVOX (editor) cannot build without licenses.json - cp /opt/voicevox_engine/engine_manifest_assets/dependency_licenses.json /opt/voicevox_engine/licenses.json -EOF - -# Create build script -RUN < /build.sh - #!/bin/bash - set -eux - - # chown general user c.z. mounted directory may be owned by root - mkdir -p /opt/voicevox_engine_build - chown -R user:user /opt/voicevox_engine_build - - mkdir -p /home/user/.cache/Nuitka - chown -R user:user /home/user/.cache/Nuitka - - cd /opt/voicevox_engine_build - - gosu user /opt/python/bin/python3 -m nuitka \ - --output-dir=/opt/voicevox_engine_build \ - --standalone \ - --plugin-enable=numpy \ - --plugin-enable=multiprocessing \ - --follow-import-to=numpy \ - --follow-import-to=aiofiles \ - --include-package=uvicorn \ - --include-package=anyio \ - --include-package-data=pyopenjtalk \ - --include-package-data=scipy \ - --include-data-file=/opt/voicevox_engine/licenses.json=./ \ - --include-data-file=/opt/voicevox_engine/presets.yaml=./ \ - --include-data-file=/opt/voicevox_engine/default.csv=./ \ - --include-data-file=/opt/voicevox_engine/engine_manifest.json=./ \ - --include-data-file=/opt/voicevox_core/*.so=./ \ - --include-data-file=/opt/onnxruntime/lib/libonnxruntime.so=./ \ - --include-data-dir=/opt/voicevox_engine/speaker_info=./speaker_info \ - --include-data-dir=/opt/voicevox_engine/engine_manifest_assets=./engine_manifest_assets \ - --follow-imports \ - --no-prefer-source-code \ - /opt/voicevox_engine/run.py - - # Copy libonnxruntime_providers_cuda.so and dependencies (CUDA/cuDNN) - if [ -f "/opt/onnxruntime/lib/libonnxruntime_providers_cuda.so" ]; then - mkdir -p /tmp/coredeps - - # Copy provider libraries (libonnxruntime.so.{version} is copied by Nuitka) - cd /opt/onnxruntime/lib - cp libonnxruntime_*.so /tmp/coredeps/ - cd - - - # assert nvidia/cuda base image - cd /usr/local/cuda/lib64 - cp libcublas.so.* libcublasLt.so.* libcudart.so.* libcufft.so.* libcurand.so.* /tmp/coredeps/ - cd - - - # remove unneed full version libraries - cd /tmp/coredeps - rm -f libcublas.so.*.* libcublasLt.so.*.* libcufft.so.*.* libcurand.so.*.* - rm -f libcudart.so.*.*.* - cd - - - # assert nvidia/cuda base image - cd /usr/lib/x86_64-linux-gnu - cp libcudnn.so.* libcudnn_*_infer.so.* /tmp/coredeps/ - cd - - - # remove unneed full version libraries - cd /tmp/coredeps - rm -f libcudnn.so.*.* libcudnn_*_infer.so.*.* - cd - - - mv /tmp/coredeps/* /opt/voicevox_engine_build/run.dist/ - rm -rf /tmp/coredeps - fi - - chmod +x /opt/voicevox_engine_build/run.dist/run -EOD - chmod +x /build.sh -EOF - -CMD [ "bash", "/build.sh" ] From 6d2d1d9cd19cde75b3e758be51d036bf759e53fc Mon Sep 17 00:00:00 2001 From: Yuto Ashida Date: Sun, 6 Nov 2022 13:27:24 +0900 Subject: [PATCH 7/9] =?UTF-8?q?PyInstaller:=20Nuitka=E3=81=AE=E3=82=AD?= =?UTF-8?q?=E3=83=A3=E3=83=83=E3=82=B7=E3=83=A5=E3=83=91=E3=82=B9=E6=8C=87?= =?UTF-8?q?=E5=AE=9A=E3=82=92=E5=89=8A=E9=99=A4=20(#506)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit remove nuitka cache path specification from makefile --- Makefile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Makefile b/Makefile index 8f18c9491..57e7b3e28 100644 --- a/Makefile +++ b/Makefile @@ -140,7 +140,6 @@ build-linux-docker-build-ubuntu20.04: .PHONY: run-linux-docker-build-ubuntu20.04 run-linux-docker-build-ubuntu20.04: docker run --rm -it \ - -v "$(shell pwd)/cache/Nuitka:/home/user/.cache/Nuitka" \ -v "$(shell pwd)/build:/opt/voicevox_engine_build" $(ARGS) \ voicevox/voicevox_engine:build-cpu-ubuntu20.04-latest $(CMD) @@ -158,7 +157,6 @@ build-linux-docker-build-nvidia-ubuntu20.04: .PHONY: run-linux-docker-build-nvidia-ubuntu20.04 run-linux-docker-build-nvidia-ubuntu20.04: docker run --rm -it \ - -v "$(shell pwd)/cache/Nuitka:/home/user/.cache/Nuitka" \ -v "$(shell pwd)/build:/opt/voicevox_engine_build" $(ARGS) \ voicevox/voicevox_engine:build-nvidia-ubuntu20.04-latest $(CMD) @@ -177,7 +175,6 @@ build-linux-docker-build-ubuntu18.04: .PHONY: run-linux-docker-build-ubuntu18.04 run-linux-docker-build-ubuntu18.04: docker run --rm -it \ - -v "$(shell pwd)/cache/Nuitka:/home/user/.cache/Nuitka" \ -v "$(shell pwd)/build:/opt/voicevox_engine_build" $(ARGS) \ voicevox/voicevox_engine:build-cpu-ubuntu18.04-latest $(CMD) @@ -195,6 +192,5 @@ build-linux-docker-build-nvidia-ubuntu18.04: .PHONY: run-linux-docker-build-nvidia-ubuntu18.04 run-linux-docker-build-nvidia-ubuntu18.04: docker run --rm -it \ - -v "$(shell pwd)/cache/Nuitka:/home/user/.cache/Nuitka" \ -v "$(shell pwd)/build:/opt/voicevox_engine_build" $(ARGS) \ voicevox/voicevox_engine:build-nvidia-ubuntu18.04-latest $(CMD) From 3f35ffd59e9abe1cd2995885e72ea9cdcba01bb5 Mon Sep 17 00:00:00 2001 From: Yuto Ashida Date: Sun, 6 Nov 2022 20:03:03 +0900 Subject: [PATCH 8/9] =?UTF-8?q?PyInstaller:=20=E3=83=93=E3=83=AB=E3=83=89?= =?UTF-8?q?=E3=83=89=E3=82=AD=E3=83=A5=E3=83=A1=E3=83=B3=E3=83=88=E3=81=A7?= =?UTF-8?q?PyInstaller=E3=82=92=E4=BD=BF=E3=81=86=E3=82=88=E3=81=86?= =?UTF-8?q?=E3=81=AB=E3=81=99=E3=82=8B=20(#505)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 29 ++++++----------------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index bcce613be..d6dfc291f 100644 --- a/README.md +++ b/README.md @@ -410,35 +410,18 @@ python make_docs.py ## ビルド -Build Tools for Visual Studio 2019 が必要です。 +この方法でビルドしたものは、リリースで公開されているものとは異なります。 +また、GPUで利用するにはcuDNNやCUDA、DirectMLなどのライブラリが追加で必要となります。 ```bash python -m pip install -r requirements-dev.txt python generate_licenses.py > licenses.json -python -m nuitka \ - --standalone \ - --plugin-enable=numpy \ - --plugin-enable=multiprocessing \ - --follow-import-to=numpy \ - --follow-import-to=aiofiles \ - --include-package=uvicorn \ - --include-package=anyio \ - --include-package-data=pyopenjtalk \ - --include-package-data=scipy \ - --include-data-file=licenses.json=./ \ - --include-data-file=presets.yaml=./ \ - --include-data-file=default.csv=./ \ - --include-data-file=engine_manifest.json=./ \ - --include-data-file=C:/path/to/cuda/*.dll=./ \ - --include-data-file=C:/path/to/onnxruntime/lib/*.dll=./ \ - --include-data-dir=.venv/Lib/site-packages/_soundfile_data=./_soundfile_data \ - --include-data-dir=speaker_info=./speaker_info \ - --msvc=14.2 \ - --follow-imports \ - --no-prefer-source-code \ - run.py +# ビルド自体はLIBCORE_PATH及びLIBONNXRUNTIME_PATHの指定がなくても可能です +LIBCORE_PATH="/path/to/libcore" \ + LIBONNXRUNTIME_PATH="/path/to/libonnxruntime" \ + pyinstaller --noconfirm run.spec ``` ## 依存関係 From c28b71d132a2dcdba57900c636673651815f0607 Mon Sep 17 00:00:00 2001 From: Yuto Ashida Date: Sun, 6 Nov 2022 20:12:56 +0900 Subject: [PATCH 9/9] =?UTF-8?q?PyInstaller:=20`engine=5Fmanifest.json`?= =?UTF-8?q?=E3=82=92=E5=90=AB=E3=81=BF=E5=BF=98=E3=82=8C=E3=81=A6=E3=81=84?= =?UTF-8?q?=E3=81=9F=E5=95=8F=E9=A1=8C=E3=81=AE=E8=A7=A3=E6=B6=88=20(#507)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit add engine manifest json --- run.spec | 1 + 1 file changed, 1 insertion(+) diff --git a/run.spec b/run.spec index f262338fb..cbf2d6cf8 100644 --- a/run.spec +++ b/run.spec @@ -6,6 +6,7 @@ import os datas = [ ('engine_manifest_assets', 'engine_manifest_assets'), ('speaker_info', 'speaker_info'), + ('engine_manifest.json', '.'), ('default.csv', '.'), ('licenses.json', '.'), ('presets.yaml', '.'),