[WIP] Link sox dynamically

.circleci/config.yml.in

@@ -241,6 +241,66 @@ jobs:
           paths:
             - ffmpeg
 
+  build_sox_linux:
+    <<: *binary_common
+    docker:
+      - image: << parameters.wheel_docker_image >>
+    resource_class: 2xlarge+
+    steps:
+      - checkout
+      - generate_cache_key
+      - restore_cache:
+          {% raw %}
+          keys:
+            - sox-linux-v0-{{ checksum ".cachekey" }}
+          {% endraw %}
+      - run:
+          command: |
+            export SOX_ROOT=${PWD}/third_party/sox
+            if [[ ! -d ${SOX_ROOT} ]]; then
+                packaging/sox/build.sh
+            fi
+      - save_cache:
+          {% raw %}
+          key: sox-linux-v0-{{ checksum ".cachekey" }}
+          {% endraw %}
+          paths:
+            - third_party/sox
+      - persist_to_workspace:
+          root: third_party
+          paths:
+            - sox
+
+  build_sox_macos:
+    <<: *binary_common
+    macos:
+      xcode: "14.0"
+    steps:
+      - checkout
+      - generate_cache_key
+      - restore_cache:
+          {% raw %}
+          keys:
+            - sox-macos-v0-{{ checksum ".cachekey" }}
+          {% endraw %}
+      - run:
+          command: |
+            export SOX_ROOT=${PWD}/third_party/sox
+            if [[ ! -d ${SOX_ROOT} ]]; then
+                brew install wget
+                packaging/sox/build.sh
+            fi
+      - save_cache:
+          {% raw %}
+          key: sox-macos-v0-{{ checksum ".cachekey" }}
+          {% endraw %}
+          paths:
+            - third_party/sox
+      - persist_to_workspace:
+          root: third_party
+          paths:
+            - sox
+
   binary_linux_wheel:
     <<: *binary_common
     docker:
@@ -253,6 +313,7 @@ jobs:
       - run:
           command: |
             export FFMPEG_ROOT=${PWD}/third_party/ffmpeg
+            export SOX_ROOT=${PWD}/third_party/sox
             packaging/build_wheel.sh
           environment:
             USE_FFMPEG: true
@@ -278,6 +339,7 @@ jobs:
           no_output_timeout: 30m
           command: |
             export FFMPEG_ROOT=${PWD}/third_party/ffmpeg
+            export SOX_ROOT=${PWD}/third_party/sox
             packaging/build_conda.sh
           environment:
             USE_FFMPEG: true
@@ -306,6 +368,7 @@ jobs:
             sh conda.sh -b
             source $HOME/miniconda3/bin/activate
             export FFMPEG_ROOT="${PWD}/third_party/ffmpeg"
+            export SOX_ROOT="${PWD}/third_party/sox"
             packaging/build_wheel.sh
           environment:
             USE_FFMPEG: true
@@ -333,6 +396,7 @@ jobs:
             source $HOME/miniconda3/bin/activate
             conda install -yq conda-build
             export FFMPEG_ROOT="${PWD}/third_party/ffmpeg"
+            export SOX_ROOT="${PWD}/third_party/sox"
             packaging/build_conda.sh
           environment:
             USE_FFMPEG: true
@@ -362,6 +426,7 @@ jobs:
             eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
             conda activate base
             export FFMPEG_ROOT="${PWD}/third_party/ffmpeg"
+            export SOX_ROOT="${PWD}/third_party/sox"
             bash packaging/build_wheel.sh
           environment:
             USE_FFMPEG: true
@@ -391,6 +456,7 @@ jobs:
             conda activate base
             conda install -yq conda-build "conda-package-handling!=1.5.0"
             export FFMPEG_ROOT="${PWD}/third_party/ffmpeg"
+            export SOX_ROOT="${PWD}/third_party/sox"
             bash packaging/build_conda.sh
           environment:
             USE_FFMPEG: true
@@ -781,13 +847,17 @@ jobs:
           command: .circleci/unittest/linux/scripts/setup_env.sh
       - run:
           name: Install torchaudio
-          command: .circleci/unittest/linux/scripts/install.sh
+          command: |
+            brew install wget pkg-config
+            .circleci/unittest/linux/scripts/install.sh
           environment:
               USE_FFMPEG: true
               USE_OPENMP: false
       - run:
           name: Run tests
-          command: .circleci/unittest/linux/scripts/run_test.sh
+          command: |
+            brew install wget pkg-config
+            .circleci/unittest/linux/scripts/run_test.sh
           environment:
               TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_APPLY_CMVN_SLIDING: true
               TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_FBANK_FEATS: true

.circleci/regenerate.py

@@ -37,6 +37,8 @@ def build_workflows(prefix="", upload=False, filter_branch=None, indentation=6):
     w += build_download_job(filter_branch)
     for os_type in ["linux", "macos", "windows"]:
         w += build_ffmpeg_job(os_type, filter_branch)
+    for os_type in ["linux", "macos"]:
+        w += build_sox_job(os_type, filter_branch)
     for btype in ["wheel", "conda"]:
         for os_type in ["linux", "macos", "windows"]:
             for python_version in PYTHON_VERSIONS:
@@ -82,6 +84,18 @@ def build_ffmpeg_job(os_type, filter_branch):
     return [{f"build_ffmpeg_{os_type}": job}]
 
 
+def build_sox_job(os_type, filter_branch):
+    job = {
+        "name": f"build_sox_{os_type}",
+        "requires": ["download_third_parties"],
+    }
+
+    if filter_branch:
+        job["filters"] = gen_filter_branch_tree(filter_branch)
+    job["python_version"] = "foo"
+    return [{f"build_sox_{os_type}": job}]
+
+
 def build_workflow_pair(btype, os_type, python_version, cu_version, filter_branch, prefix="", upload=False):
 
     w = []
@@ -156,6 +170,9 @@ def generate_base_workflow(base_workflow_name, python_version, cu_version, filte
         "requires": [f"build_ffmpeg_{os_type}"],
     }
 
+    if os_type in ["linux", "macos"]:
+        d["requires"].append(f"build_sox_{os_type}")
+
     if btype == "conda":
         d["conda_docker_image"] = f'pytorch/conda-builder:{cu_version.replace("cu1","cuda1")}'
     elif cu_version.startswith("cu"):

.circleci/unittest/linux/scripts/install.sh

@@ -18,6 +18,93 @@ case "$(uname -s)" in
     *) os=Linux
 esac
 
+
+# if [ "${os}" != MacOSX ] ; then
+    # apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A4B469963BF863CC
+    # apt-get -qq update
+    # apt-get -y install libopencore-amrnb-dev libopencore-amrwb-dev libflac-dev libvorbis-dev libmp3lame-dev opus-tools
+
+wget https://sourceforge.net/projects/opencore-amr/files/opencore-amr/opencore-amr-0.1.5.tar.gz
+tar -xf opencore-amr-0.1.5.tar.gz
+cd opencore-amr-0.1.5
+./configure
+make install
+cd ..
+
+wget https://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz
+tar -xf lame-3.99.5.tar.gz
+cd lame-3.99.5
+./configure
+make install
+cd ..
+
+wget https://ftp.osuosl.org/pub/xiph/releases/ogg/libogg-1.3.3.tar.gz
+tar -xf libogg-1.3.3.tar.gz
+cd libogg-1.3.3
+./configure
+make install
+cd ..
+
+wget https://ftp.osuosl.org/pub/xiph/releases/flac/flac-1.3.2.tar.xz
+tar -xf flac-1.3.2.tar.xz
+cd flac-1.3.2
+./configure --with-ogg --disable-cpplibs
+make install
+cd ..
+
+wget https://ftp.osuosl.org/pub/xiph/releases/vorbis/libvorbis-1.3.6.tar.gz
+tar -xf libvorbis-1.3.6.tar.gz
+cd libvorbis-1.3.6
+./configure --with-ogg
+make install
+cd ..
+
+wget https://ftp.osuosl.org/pub/xiph/releases/opus/opus-1.3.1.tar.gz
+tar -xf opus-1.3.1.tar.gz
+cd opus-1.3.1
+./configure
+make install
+cd ..
+
+wget https://ftp.osuosl.org/pub/xiph/releases/opus/opusfile-0.12.tar.gz
+tar -xf opusfile-0.12.tar.gz
+cd opusfile-0.12
+./configure --disable-http
+make install
+cd ..
+
+wget https://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2
+tar -xf sox-14.4.2.tar.bz2
+cd sox-14.4.2
+./configure \
+    --disable-openmp \
+    --with-amrnb \
+    --with-amrwb \
+    --with-flac \
+    --with-lame \
+    --with-oggvorbis \
+    --with-opus \
+    --without-alsa \
+    --without-ao \
+    --without-coreaudio \
+    --without-oss \
+    --without-id3tag \
+    --without-ladspa \
+    --without-mad \
+    --without-magic \
+    --without-png \
+    --without-pulseaudio \
+    --without-sndfile \
+    --without-sndio \
+    --without-sunaudio \
+    --without-waveaudio \
+    --without-wavpack \
+    --without-twolame
+make install
+cd ..
+export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib"
+# fi
+
 # 0. Activate conda env
 eval "$("${conda_dir}/bin/conda" shell.bash hook)"
 conda activate "${env_dir}"

.circleci/unittest/linux/scripts/run_test.sh

@@ -2,6 +2,94 @@
 
 set -e
 
+
+# if [ "${os}" != MacOSX ] ; then
+    # apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A4B469963BF863CC
+    # apt-get -qq update
+
+    # apt-get -y install libopencore-amrnb-dev libopencore-amrwb-dev libflac-dev libvorbis-dev libmp3lame-dev opus-tools
+
+wget https://sourceforge.net/projects/opencore-amr/files/opencore-amr/opencore-amr-0.1.5.tar.gz
+tar -xf opencore-amr-0.1.5.tar.gz
+cd opencore-amr-0.1.5
+./configure
+make install
+cd ..
+
+wget https://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz
+tar -xf lame-3.99.5.tar.gz
+cd lame-3.99.5
+./configure
+make install
+cd ..
+
+wget https://ftp.osuosl.org/pub/xiph/releases/ogg/libogg-1.3.3.tar.gz
+tar -xf libogg-1.3.3.tar.gz
+cd libogg-1.3.3
+./configure
+make install
+cd ..
+
+wget https://ftp.osuosl.org/pub/xiph/releases/flac/flac-1.3.2.tar.xz
+tar -xf flac-1.3.2.tar.xz
+cd flac-1.3.2
+./configure --with-ogg --disable-cpplibs
+make install
+cd ..
+
+wget https://ftp.osuosl.org/pub/xiph/releases/vorbis/libvorbis-1.3.6.tar.gz
+tar -xf libvorbis-1.3.6.tar.gz
+cd libvorbis-1.3.6
+./configure --with-ogg
+make install
+cd ..
+
+wget https://ftp.osuosl.org/pub/xiph/releases/opus/opus-1.3.1.tar.gz
+tar -xf opus-1.3.1.tar.gz
+cd opus-1.3.1
+./configure
+make install
+cd ..
+
+wget https://ftp.osuosl.org/pub/xiph/releases/opus/opusfile-0.12.tar.gz
+tar -xf opusfile-0.12.tar.gz
+cd opusfile-0.12
+./configure --disable-http
+make install
+cd ..
+
+wget https://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2
+tar -xf sox-14.4.2.tar.bz2
+cd sox-14.4.2
+./configure \
+    --disable-openmp \
+    --with-amrnb \
+    --with-amrwb \
+    --with-flac \
+    --with-lame \
+    --with-oggvorbis \
+    --with-opus \
+    --without-alsa \
+    --without-ao \
+    --without-coreaudio \
+    --without-oss \
+    --without-id3tag \
+    --without-ladspa \
+    --without-mad \
+    --without-magic \
+    --without-png \
+    --without-pulseaudio \
+    --without-sndfile \
+    --without-sndio \
+    --without-sunaudio \
+    --without-waveaudio \
+    --without-wavpack \
+    --without-twolame
+make install
+cd ..
+export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib"
+# fi
+
 eval "$(./conda/bin/conda shell.bash hook)"
 conda activate ./env
 

.github/workflows/integration-test.yml

@@ -24,7 +24,7 @@ jobs:
     - name: Install dependencies
       run: |
         sudo add-apt-repository -y ppa:jonathonf/ffmpeg-4
-        sudo apt install -y -qq pkg-config libavfilter-dev libavdevice-dev
+        sudo apt install -y -qq pkg-config libavfilter-dev libavdevice-dev sox libsox-dev libsox-fmt-all
     - name: Install packages
       run: |
         python -m pip install --quiet --upgrade pip

CMakeLists.txt

@@ -56,7 +56,7 @@ endif()
 
 
 # Options
-option(BUILD_SOX "Build libsox statically" ON)
+option(USE_SOX "Enable SoX features" ON)
 option(BUILD_KALDI "Build kaldi statically" ON)
 option(BUILD_RNNT "Enable RNN transducer" ON)
 option(BUILD_CTC_DECODER "Build Flashlight CTC decoder" ON)