diff --git a/.circleci/config.yml b/.circleci/config.yml index ee442189e4..d51bb3ca96 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -101,14 +101,14 @@ jobs: - run: command: | - mkdir -p third_party/archives/ - wget --no-clobber --directory-prefix=third_party/archives/ $(awk '/URL /{print $2}' third_party/CMakeLists.txt) + mkdir -p third_party/sox/archives/ + wget --no-clobber --directory-prefix=third_party/sox/archives/ $(awk '/URL /{print $2}' third_party/sox/CMakeLists.txt) - save_cache: key: tp-nix-v2-{{ checksum ".cachekey" }} paths: - - third_party/archives + - third_party/sox/archives - persist_to_workspace: root: third_party paths: @@ -411,9 +411,7 @@ jobs: paths: - conda - env - - third_party/build - third_party/install - - third_party/src - run: name: Install torchaudio command: .circleci/unittest/linux/scripts/install.sh @@ -458,9 +456,7 @@ jobs: paths: - conda - env - - third_party/build - third_party/install - - third_party/src - run: name: Install torchaudio command: docker run -t --gpus all -e UPLOAD_CHANNEL -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh @@ -573,9 +569,7 @@ jobs: paths: - conda - env - - third_party/build - third_party/install - - third_party/src - run: name: Install torchaudio command: .circleci/unittest/linux/scripts/install.sh @@ -612,9 +606,7 @@ jobs: paths: - conda - env - - third_party/build - third_party/install - - third_party/src - run: name: Run style check command: .circleci/unittest/linux/scripts/run_style_checks.sh diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 5c9c4d7693..9f6f41a379 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -101,14 +101,14 @@ jobs: {% endraw %} - run: command: | - mkdir -p third_party/archives/ - wget --no-clobber --directory-prefix=third_party/archives/ $(awk '/URL /{print $2}' third_party/CMakeLists.txt) + mkdir -p third_party/sox/archives/ + wget --no-clobber --directory-prefix=third_party/sox/archives/ $(awk '/URL /{print $2}' third_party/sox/CMakeLists.txt) - save_cache: {% raw %} key: tp-nix-v2-{{ checksum ".cachekey" }} {% endraw %} paths: - - third_party/archives + - third_party/sox/archives - persist_to_workspace: root: third_party paths: @@ -411,9 +411,7 @@ jobs: paths: - conda - env - - third_party/build - third_party/install - - third_party/src - run: name: Install torchaudio command: .circleci/unittest/linux/scripts/install.sh @@ -458,9 +456,7 @@ jobs: paths: - conda - env - - third_party/build - third_party/install - - third_party/src - run: name: Install torchaudio command: docker run -t --gpus all -e UPLOAD_CHANNEL -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh @@ -573,9 +569,7 @@ jobs: paths: - conda - env - - third_party/build - third_party/install - - third_party/src - run: name: Install torchaudio command: .circleci/unittest/linux/scripts/install.sh @@ -612,9 +606,7 @@ jobs: paths: - conda - env - - third_party/build - third_party/install - - third_party/src - run: name: Run style check command: .circleci/unittest/linux/scripts/run_style_checks.sh diff --git a/.gitignore b/.gitignore index e8a9d4c409..ae628691f5 100644 --- a/.gitignore +++ b/.gitignore @@ -123,7 +123,5 @@ examples/interactive_asr/data/*.model examples/interactive_asr/data/*.pt # third parties -third_party/archives/ third_party/install/ -third_party/src/ -third_party/tmp/ +third_party/sox/archives/ diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt index 14dbd120a6..1284affa88 100644 --- a/third_party/CMakeLists.txt +++ b/third_party/CMakeLists.txt @@ -8,94 +8,9 @@ option(BUILD_TRANSDUCER "Build transducer statically") set(CMAKE_POSITION_INDEPENDENT_CODE ON) if (BUILD_SOX) -include(ExternalProject) -set(INSTALL_DIR ${CMAKE_INSTALL_PREFIX}) -set(ARCHIVE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/archives) -set(COMMON_ARGS --quiet --disable-shared --enable-static --prefix=${INSTALL_DIR} --with-pic --disable-dependency-tracking --disable-debug --disable-examples --disable-doc) - -ExternalProject_Add(libmad - PREFIX ${CMAKE_CURRENT_SOURCE_DIR} - DOWNLOAD_DIR ${ARCHIVE_DIR} - URL https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz - URL_HASH SHA256=bbfac3ed6bfbc2823d3775ebb931087371e142bb0e9bb1bee51a76a6e0078690 - PATCH_COMMAND patch < ${CMAKE_CURRENT_SOURCE_DIR}/patch/libmad.patch - CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/libmad/configure ${COMMON_ARGS} -) - -ExternalProject_Add(amr - PREFIX ${CMAKE_CURRENT_SOURCE_DIR} - DOWNLOAD_DIR ${ARCHIVE_DIR} - URL https://sourceforge.net/projects/opencore-amr/files/opencore-amr/opencore-amr-0.1.5.tar.gz - URL_HASH SHA256=2c006cb9d5f651bfb5e60156dbff6af3c9d35c7bbcc9015308c0aff1e14cd341 - CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/amr/configure ${COMMON_ARGS} -) - -ExternalProject_Add(libmp3lame - PREFIX ${CMAKE_CURRENT_SOURCE_DIR} - DOWNLOAD_DIR ${ARCHIVE_DIR} - URL https://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz - URL_HASH SHA256=24346b4158e4af3bd9f2e194bb23eb473c75fb7377011523353196b19b9a23ff - CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/libmp3lame/configure ${COMMON_ARGS} --enable-nasm -) - -ExternalProject_Add(libogg - PREFIX ${CMAKE_CURRENT_SOURCE_DIR} - DOWNLOAD_DIR ${ARCHIVE_DIR} - URL https://ftp.osuosl.org/pub/xiph/releases/ogg/libogg-1.3.3.tar.gz - URL_HASH SHA256=c2e8a485110b97550f453226ec644ebac6cb29d1caef2902c007edab4308d985 - CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/libogg/configure ${COMMON_ARGS} -) - -ExternalProject_Add(libflac - PREFIX ${CMAKE_CURRENT_SOURCE_DIR} - DEPENDS libogg - DOWNLOAD_DIR ${ARCHIVE_DIR} - URL https://ftp.osuosl.org/pub/xiph/releases/flac/flac-1.3.2.tar.xz - URL_HASH SHA256=91cfc3ed61dc40f47f050a109b08610667d73477af6ef36dcad31c31a4a8d53f - CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/build_codec_helper.sh ${CMAKE_CURRENT_SOURCE_DIR}/src/libflac/configure ${COMMON_ARGS} --with-ogg -) - -ExternalProject_Add(libvorbis - PREFIX ${CMAKE_CURRENT_SOURCE_DIR} - DEPENDS libogg - DOWNLOAD_DIR ${ARCHIVE_DIR} - URL https://ftp.osuosl.org/pub/xiph/releases/vorbis/libvorbis-1.3.6.tar.gz - URL_HASH SHA256=6ed40e0241089a42c48604dc00e362beee00036af2d8b3f46338031c9e0351cb - CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/build_codec_helper.sh ${CMAKE_CURRENT_SOURCE_DIR}/src/libvorbis/configure ${COMMON_ARGS} --with-ogg -) - -ExternalProject_Add(libopus - PREFIX ${CMAKE_CURRENT_SOURCE_DIR} - DEPENDS libogg - DOWNLOAD_DIR ${ARCHIVE_DIR} - URL https://ftp.osuosl.org/pub/xiph/releases/opus/opus-1.3.1.tar.gz - URL_HASH SHA256=65b58e1e25b2a114157014736a3d9dfeaad8d41be1c8179866f144a2fb44ff9d - CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/build_codec_helper.sh ${CMAKE_CURRENT_SOURCE_DIR}/src/libopus/configure ${COMMON_ARGS} --with-ogg -) - -ExternalProject_Add(opusfile - PREFIX ${CMAKE_CURRENT_SOURCE_DIR} - DEPENDS libopus - DOWNLOAD_DIR ${ARCHIVE_DIR} - STAMP_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/opusfile-stamp - SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/opusfile - URL https://ftp.osuosl.org/pub/xiph/releases/opus/opusfile-0.12.tar.gz - URL_HASH SHA256=118d8601c12dd6a44f52423e68ca9083cc9f2bfe72da7a8c1acb22a80ae3550b - CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/build_codec_helper.sh ${CMAKE_CURRENT_SOURCE_DIR}/src/opusfile/configure ${COMMON_ARGS} --disable-http -) - -ExternalProject_Add(libsox - PREFIX ${CMAKE_CURRENT_SOURCE_DIR} - DEPENDS libogg libflac libvorbis opusfile libmp3lame libmad amr - DOWNLOAD_DIR ${ARCHIVE_DIR} - URL https://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2 - URL_HASH SHA256=81a6956d4330e75b5827316e44ae381e6f1e8928003c6aa45896da9041ea149c - # OpenMP is by default compiled against GNU OpenMP, which conflicts with the version of OpenMP that PyTorch uses. - # See https://github.com/pytorch/audio/pull/1026 - CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/build_codec_helper.sh ${CMAKE_CURRENT_SOURCE_DIR}/src/libsox/configure ${COMMON_ARGS} --with-lame --with-flac --with-mad --with-oggvorbis --without-alsa --without-coreaudio --without-png --without-oss --without-sndfile --with-opus --with-amrwb --with-amrnb --disable-openmp --without-sndio --without-pulseaudio -) + add_subdirectory(sox) endif() if(BUILD_TRANSDUCER) -add_subdirectory(transducer) + add_subdirectory(transducer) endif() diff --git a/third_party/build_codec_helper.sh b/third_party/build_codec_helper.sh deleted file mode 100755 index e7f2614781..0000000000 --- a/third_party/build_codec_helper.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -# Helper script for building codecs depending on libogg, such as libopus and opus. -# It is difficult to set environment variable inside of ExternalProject_Add, -# so this script sets necessary environment variables before running the given command - -this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -install_dir="${this_dir}/install" - -export PKG_CONFIG_PATH="${install_dir}/lib/pkgconfig" -export LDFLAGS="-L${install_dir}/lib ${LDFLAGS}" -export CPPFLAGS="-I${install_dir}/include ${CPPFLAGS}" - -$@ diff --git a/third_party/sox/CMakeLists.txt b/third_party/sox/CMakeLists.txt new file mode 100644 index 0000000000..dcad72bbfe --- /dev/null +++ b/third_party/sox/CMakeLists.txt @@ -0,0 +1,95 @@ +include(ExternalProject) + +set(INSTALL_DIR ${PROJECT_SOURCE_DIR}/install) +set(ARCHIVE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/archives) +set(COMMON_ARGS --quiet --disable-shared --enable-static --prefix=${INSTALL_DIR} --with-pic --disable-dependency-tracking --disable-debug --disable-examples --disable-doc) + +# To pass custom environment variables to ExternalProject_Add command, +# we need to do `${CMAKE_COMMAND} -E env ${envs} `. +# https://stackoverflow.com/a/62437353 +# We constrcut the custom environment variables here +set(envs + "PKG_CONFIG_PATH=${INSTALL_DIR}/lib/pkgconfig" + "LDFLAGS=-L${INSTALL_DIR}/lib $ENV{LDFLAGS}" + "CPPFLAGS=-I${INSTALL_DIR}/include $ENV{CPPFLAGS}" +) + +ExternalProject_Add(mad + PREFIX ${CMAKE_CURRENT_BINARY_DIR} + DOWNLOAD_DIR ${ARCHIVE_DIR} + URL https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz + URL_HASH SHA256=bbfac3ed6bfbc2823d3775ebb931087371e142bb0e9bb1bee51a76a6e0078690 + PATCH_COMMAND patch < ${CMAKE_CURRENT_SOURCE_DIR}/patch/libmad.patch + CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/mad/configure ${COMMON_ARGS} +) + +ExternalProject_Add(amr + PREFIX ${CMAKE_CURRENT_BINARY_DIR} + DOWNLOAD_DIR ${ARCHIVE_DIR} + URL https://sourceforge.net/projects/opencore-amr/files/opencore-amr/opencore-amr-0.1.5.tar.gz + URL_HASH SHA256=2c006cb9d5f651bfb5e60156dbff6af3c9d35c7bbcc9015308c0aff1e14cd341 + CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/amr/configure ${COMMON_ARGS} +) + +ExternalProject_Add(lame + PREFIX ${CMAKE_CURRENT_BINARY_DIR} + DOWNLOAD_DIR ${ARCHIVE_DIR} + URL https://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz + URL_HASH SHA256=24346b4158e4af3bd9f2e194bb23eb473c75fb7377011523353196b19b9a23ff + CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/lame/configure ${COMMON_ARGS} --enable-nasm +) + +ExternalProject_Add(ogg + PREFIX ${CMAKE_CURRENT_BINARY_DIR} + DOWNLOAD_DIR ${ARCHIVE_DIR} + URL https://ftp.osuosl.org/pub/xiph/releases/ogg/libogg-1.3.3.tar.gz + URL_HASH SHA256=c2e8a485110b97550f453226ec644ebac6cb29d1caef2902c007edab4308d985 + CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/ogg/configure ${COMMON_ARGS} +) + +ExternalProject_Add(flac + PREFIX ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS ogg + DOWNLOAD_DIR ${ARCHIVE_DIR} + URL https://ftp.osuosl.org/pub/xiph/releases/flac/flac-1.3.2.tar.xz + URL_HASH SHA256=91cfc3ed61dc40f47f050a109b08610667d73477af6ef36dcad31c31a4a8d53f + CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/flac/configure ${COMMON_ARGS} --with-ogg --disable-cpplibs +) + +ExternalProject_Add(vorbis + PREFIX ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS ogg + DOWNLOAD_DIR ${ARCHIVE_DIR} + URL https://ftp.osuosl.org/pub/xiph/releases/vorbis/libvorbis-1.3.6.tar.gz + URL_HASH SHA256=6ed40e0241089a42c48604dc00e362beee00036af2d8b3f46338031c9e0351cb + CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/vorbis/configure ${COMMON_ARGS} --with-ogg +) + +ExternalProject_Add(opus + PREFIX ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS ogg + DOWNLOAD_DIR ${ARCHIVE_DIR} + URL https://ftp.osuosl.org/pub/xiph/releases/opus/opus-1.3.1.tar.gz + URL_HASH SHA256=65b58e1e25b2a114157014736a3d9dfeaad8d41be1c8179866f144a2fb44ff9d + CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/opus/configure ${COMMON_ARGS} --with-ogg +) + +ExternalProject_Add(opusfile + PREFIX ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS opus + DOWNLOAD_DIR ${ARCHIVE_DIR} + URL https://ftp.osuosl.org/pub/xiph/releases/opus/opusfile-0.12.tar.gz + URL_HASH SHA256=118d8601c12dd6a44f52423e68ca9083cc9f2bfe72da7a8c1acb22a80ae3550b + CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/opusfile/configure ${COMMON_ARGS} --disable-http +) + +ExternalProject_Add(sox + PREFIX ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS ogg flac vorbis opusfile lame mad amr + DOWNLOAD_DIR ${ARCHIVE_DIR} + URL https://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2 + URL_HASH SHA256=81a6956d4330e75b5827316e44ae381e6f1e8928003c6aa45896da9041ea149c + # OpenMP is by default compiled against GNU OpenMP, which conflicts with the version of OpenMP that PyTorch uses. + # See https://github.com/pytorch/audio/pull/1026 + CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/sox/configure ${COMMON_ARGS} --with-lame --with-flac --with-mad --with-oggvorbis --without-alsa --without-coreaudio --without-png --without-oss --without-sndfile --with-opus --with-amrwb --with-amrnb --disable-openmp --without-sndio --without-pulseaudio +) diff --git a/third_party/patch/libmad.patch b/third_party/sox/patch/libmad.patch similarity index 100% rename from third_party/patch/libmad.patch rename to third_party/sox/patch/libmad.patch