Link sox dynamically

pytorch · Oct 21, 2022 · 60e25c7 · 60e25c7
1 parent 5239583
commit 60e25c7
Show file tree

Hide file tree

Showing 15 changed files with 554 additions and 234 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in
@@ -241,6 +241,66 @@ jobs:
           paths:
             - ffmpeg
 
+  build_sox_linux:
+    <<: *binary_common
+    docker:
+      - image: << parameters.wheel_docker_image >>
+    resource_class: 2xlarge+
+    steps:
+      - checkout
+      - generate_cache_key
+      - restore_cache:
+          {% raw %}
+          keys:
+            - sox-linux-v0-{{ checksum ".cachekey" }}
+          {% endraw %}
+      - run:
+          command: |
+            export SOX_ROOT=${PWD}/third_party/sox
+            if [[ ! -d ${SOX_ROOT} ]]; then
+                packaging/sox/build.sh
+            fi
+      - save_cache:
+          {% raw %}
+          key: sox-linux-v0-{{ checksum ".cachekey" }}
+          {% endraw %}
+          paths:
+            - third_party/sox
+      - persist_to_workspace:
+          root: third_party
+          paths:
+            - sox
+
+  build_sox_macos:
+    <<: *binary_common
+    macos:
+      xcode: "14.0"
+    steps:
+      - checkout
+      - generate_cache_key
+      - restore_cache:
+          {% raw %}
+          keys:
+            - sox-macos-v0-{{ checksum ".cachekey" }}
+          {% endraw %}
+      - run:
+          command: |
+            export SOX_ROOT=${PWD}/third_party/sox
+            if [[ ! -d ${SOX_ROOT} ]]; then
+                brew install wget
+                packaging/sox/build.sh
+            fi
+      - save_cache:
+          {% raw %}
+          key: sox-macos-v0-{{ checksum ".cachekey" }}
+          {% endraw %}
+          paths:
+            - third_party/sox
+      - persist_to_workspace:
+          root: third_party
+          paths:
+            - sox
+
   binary_linux_wheel:
     <<: *binary_common
     docker:
@@ -253,6 +313,7 @@ jobs:
       - run:
           command: |
             export FFMPEG_ROOT=${PWD}/third_party/ffmpeg
+            export SOX_ROOT=${PWD}/third_party/sox
             packaging/build_wheel.sh
           environment:
             USE_FFMPEG: true
@@ -278,6 +339,7 @@ jobs:
           no_output_timeout: 30m
           command: |
             export FFMPEG_ROOT=${PWD}/third_party/ffmpeg
+            export SOX_ROOT=${PWD}/third_party/sox
             packaging/build_conda.sh
           environment:
             USE_FFMPEG: true
@@ -306,6 +368,7 @@ jobs:
             sh conda.sh -b
             source $HOME/miniconda3/bin/activate
             export FFMPEG_ROOT="${PWD}/third_party/ffmpeg"
+            export SOX_ROOT="${PWD}/third_party/sox"
             packaging/build_wheel.sh
           environment:
             USE_FFMPEG: true
@@ -333,6 +396,7 @@ jobs:
             source $HOME/miniconda3/bin/activate
             conda install -yq conda-build
             export FFMPEG_ROOT="${PWD}/third_party/ffmpeg"
+            export SOX_ROOT="${PWD}/third_party/sox"
             packaging/build_conda.sh
           environment:
             USE_FFMPEG: true
@@ -362,6 +426,7 @@ jobs:
             eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
             conda activate base
             export FFMPEG_ROOT="${PWD}/third_party/ffmpeg"
+            export SOX_ROOT="${PWD}/third_party/sox"
             bash packaging/build_wheel.sh
           environment:
             USE_FFMPEG: true
@@ -391,6 +456,7 @@ jobs:
             conda activate base
             conda install -yq conda-build "conda-package-handling!=1.5.0"
             export FFMPEG_ROOT="${PWD}/third_party/ffmpeg"
+            export SOX_ROOT="${PWD}/third_party/sox"
             bash packaging/build_conda.sh
           environment:
             USE_FFMPEG: true

diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py
@@ -37,6 +37,8 @@ def build_workflows(prefix="", upload=False, filter_branch=None, indentation=6):
     w += build_download_job(filter_branch)
     for os_type in ["linux", "macos", "windows"]:
         w += build_ffmpeg_job(os_type, filter_branch)
+    for os_type in ["linux", "macos"]:
+        w += build_sox_job(os_type, filter_branch)
     for btype in ["wheel", "conda"]:
         for os_type in ["linux", "macos", "windows"]:
             for python_version in PYTHON_VERSIONS:
@@ -82,6 +84,18 @@ def build_ffmpeg_job(os_type, filter_branch):
     return [{f"build_ffmpeg_{os_type}": job}]
 
 
+def build_sox_job(os_type, filter_branch):
+    job = {
+        "name": f"build_sox_{os_type}",
+        "requires": ["download_third_parties"],
+    }
+
+    if filter_branch:
+        job["filters"] = gen_filter_branch_tree(filter_branch)
+    job["python_version"] = "foo"
+    return [{f"build_sox_{os_type}": job}]
+
+
 def build_workflow_pair(btype, os_type, python_version, cu_version, filter_branch, prefix="", upload=False):
 
     w = []
@@ -156,6 +170,9 @@ def generate_base_workflow(base_workflow_name, python_version, cu_version, filte
         "requires": [f"build_ffmpeg_{os_type}"],
     }
 
+    if os_type in ["linux", "macos"]:
+        d["requires"].append(f"build_sox_{os_type}")
+
     if btype == "conda":
         d["conda_docker_image"] = f'pytorch/conda-builder:{cu_version.replace("cu1","cuda1")}'
     elif cu_version.startswith("cu"):

diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh
@@ -18,6 +18,58 @@ case "$(uname -s)" in
     *) os=Linux
 esac
 
+
+if [ "${os}" != MacOSX ] ; then
+    apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A4B469963BF863CC
+    apt-get -qq update
+    apt-get -y install libopencore-amrnb-dev libopencore-amrwb-dev libflac-dev libvorbis-dev libmp3lame-dev opus-tools
+
+    wget https://ftp.osuosl.org/pub/xiph/releases/opus/opus-1.3.1.tar.gz
+    tar -xf opus-1.3.1.tar.gz
+    cd opus-1.3.1
+    ./configure
+    make install
+    cd ..
+
+    wget https://ftp.osuosl.org/pub/xiph/releases/opus/opusfile-0.12.tar.gz
+    tar -xf opusfile-0.12.tar.gz
+    cd opusfile-0.12
+    ./configure --disable-http
+    make install
+    cd ..
+
+    wget https://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2
+    tar -xf sox-14.4.2.tar.bz2
+    cd sox-14.4.2
+    ./configure \
+        --disable-openmp \
+        --with-amrnb \
+        --with-amrwb \
+        --with-flac \
+        --with-lame \
+        --with-oggvorbis \
+        --with-opus \
+        --without-alsa \
+        --without-ao \
+        --without-coreaudio \
+        --without-oss \
+        --without-id3tag \
+        --without-ladspa \
+        --without-mad \
+        --without-magic \
+        --without-png \
+        --without-pulseaudio \
+        --without-sndfile \
+        --without-sndio \
+        --without-sunaudio \
+        --without-waveaudio \
+        --without-wavpack \
+        --without-twolame
+    make install
+    cd ..
+    export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib"
+fi
+
 # 0. Activate conda env
 eval "$("${conda_dir}/bin/conda" shell.bash hook)"
 conda activate "${env_dir}"

diff --git a/.circleci/unittest/linux/scripts/run_test.sh b/.circleci/unittest/linux/scripts/run_test.sh
@@ -2,6 +2,94 @@
 
 set -e
 
+
+if [ "${os}" != MacOSX ] ; then
+    # apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A4B469963BF863CC
+    # apt-get -qq update
+
+    # apt-get -y install libopencore-amrnb-dev libopencore-amrwb-dev libflac-dev libvorbis-dev libmp3lame-dev opus-tools
+
+    wget https://sourceforge.net/projects/opencore-amr/files/opencore-amr/opencore-amr-0.1.5.tar.gz
+    tar -xf opencore-amr-0.1.5.tar.gz
+    cd opencore-amr-0.1.5
+    ./configure
+    make install
+    cd ..
+
+    wget https://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz
+    tar -xf lame-3.99.5.tar.gz
+    cd lame-3.99.5
+    ./configure
+    make install
+    cd ..
+
+    wget https://ftp.osuosl.org/pub/xiph/releases/ogg/libogg-1.3.3.tar.gz
+    tar -xf libogg-1.3.3.tar.gz
+    cd libogg-1.3.3
+    ./configure
+    make install
+    cd ..
+
+    wget https://ftp.osuosl.org/pub/xiph/releases/flac/flac-1.3.2.tar.xz
+    tar -xf flac-1.3.2.tar.xz
+    cd flac-1.3.2
+    ./configure --with-ogg --disable-cpplibs
+    make install
+    cd ..
+
+    wget https://ftp.osuosl.org/pub/xiph/releases/vorbis/libvorbis-1.3.6.tar.gz
+    tar -xf libvorbis-1.3.6.tar.gz
+    cd libvorbis-1.3.6
+    ./configure --with-ogg
+    make install
+    cd ..
+
+    wget https://ftp.osuosl.org/pub/xiph/releases/opus/opus-1.3.1.tar.gz
+    tar -xf opus-1.3.1.tar.gz
+    cd opus-1.3.1
+    ./configure
+    make install
+    cd ..
+
+    wget https://ftp.osuosl.org/pub/xiph/releases/opus/opusfile-0.12.tar.gz
+    tar -xf opusfile-0.12.tar.gz
+    cd opusfile-0.12
+    ./configure --disable-http
+    make install
+    cd ..
+
+    wget https://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2
+    tar -xf sox-14.4.2.tar.bz2
+    cd sox-14.4.2
+    ./configure \
+        --disable-openmp \
+        --with-amrnb \
+        --with-amrwb \
+        --with-flac \
+        --with-lame \
+        --with-oggvorbis \
+        --with-opus \
+        --without-alsa \
+        --without-ao \
+        --without-coreaudio \
+        --without-oss \
+        --without-id3tag \
+        --without-ladspa \
+        --without-mad \
+        --without-magic \
+        --without-png \
+        --without-pulseaudio \
+        --without-sndfile \
+        --without-sndio \
+        --without-sunaudio \
+        --without-waveaudio \
+        --without-wavpack \
+        --without-twolame
+    make install
+    cd ..
+    export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib"
+fi
+
 eval "$(./conda/bin/conda shell.bash hook)"
 conda activate ./env
 

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
@@ -24,7 +24,7 @@ jobs:
     - name: Install dependencies
       run: |
         sudo add-apt-repository -y ppa:jonathonf/ffmpeg-4
-        sudo apt install -y -qq pkg-config libavfilter-dev libavdevice-dev
+        sudo apt install -y -qq pkg-config libavfilter-dev libavdevice-dev sox libsox-dev libsox-fmt-all
     - name: Install packages
       run: |
         python -m pip install --quiet --upgrade pip

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -56,7 +56,7 @@ endif()
 
 
 # Options
-option(BUILD_SOX "Build libsox statically" ON)
+option(USE_SOX "Enable SoX features" ON)
 option(BUILD_KALDI "Build kaldi statically" ON)
 option(BUILD_RNNT "Enable RNN transducer" ON)
 option(BUILD_CTC_DECODER "Build Flashlight CTC decoder" ON)

diff --git a/cmake/FindSOX.cmake b/cmake/FindSOX.cmake
@@ -0,0 +1,40 @@
+if (NOT SOX_FOUND)
+  find_package(PkgConfig)
+  if (PKG_CONFIG_FOUND)
+    pkg_check_modules(SOX sox)
+    if (SOX_FOUND)
+      message(STATUS "PkgConfig found sox (include: ${SOX_INCLUDE_DIRS}, link libraries: ${SOX_LINK_LIBRARIES})")
+    endif()
+  endif()
+
+  if (NOT SOX_FOUND)
+    set(SOX_ROOT $ENV{SOX_ROOT} CACHE PATH "Folder contains the sox library")
+
+    find_path(SOX_INCLUDE_DIRS
+      NAMES sox.h
+      PATHS ${SOX_INCLUDE_DIR} ${SOX_ROOT}
+      PATH_SUFFIXES "include" "include/sox/"
+      )
+
+    if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+      set(SOX_LIBNAME "libsox.dylib")
+    else()
+      SET(SOX_LIBNAME "libsox${CMAKE_SHARED_LIBRARY_SUFFIX}")
+    endif()
+
+    find_library(SOX_LINK_LIBRARIES
+      NAMES ${SOX_LIBNAME}
+      PATHS ${SOX_LIB_DIR} ${SOX_ROOT}
+      PATH_SUFFIXES "lib" "lib64"
+      )
+
+    include(FindPackageHandleStandardArgs)
+    find_package_handle_standard_args(SOX DEFAULT_MSG SOX_INCLUDE_DIRS SOX_LINK_LIBRARIES)
+
+    if(SOX_FOUND)
+      set (SOX_HEADER_FILE "${SOX_INCLUDE_DIRS}/sox.h")
+      message(STATUS "Found SOX (include: ${SOX_INCLUDE_DIRS}, library: ${SOX_LINK_LIBRARIES})")
+      mark_as_advanced(SOX_ROOT SOX_INCLUDE_DIRS SOX_LINK_LIBRARIES)
+    endif()
+  endif()
+endif()