Replace libsox with stub library (pytorch#3497)

Summary:
This commit updates the way libsox is integrated to torchaudio

1. We stop statically linking libsox, so torchaudio will not ship libsox.
2. We link libsox dynamically. Users are expected to install libsox by themselves.
3. We use stab library to build torchaudio.

Pull Request resolved: pytorch#3497

Differential Revision: D47803706

Pulled By: mthrok

fbshipit-source-id: 31b05495d81069186fa52d67beea360cc7e817a8

.github/scripts/unittest-linux/install.sh

@@ -75,7 +75,7 @@ fi
 # Note: installing librosa via pip fail because it will try to compile numba.
 (
     set -x
-    conda install -y -c conda-forge ${NUMBA_DEV_CHANNEL} 'librosa==0.10.0' parameterized 'requests>=2.20' 'ffmpeg>=5,<7'
+    conda install -y -c conda-forge ${NUMBA_DEV_CHANNEL} sox libvorbis 'librosa==0.10.0' parameterized 'requests>=2.20' 'ffmpeg>=5,<7'
     pip install kaldi-io SoundFile coverage pytest pytest-cov 'scipy==1.7.3' expecttest unidecode inflect Pillow sentencepiece pytorch-lightning 'protobuf<4.21.0' demucs tinytag pyroomacoustics flashlight-text git+https://github.com/kpu/kenlm
 )
 # Install fairseq

.github/workflows/build_docs.yml

@@ -63,7 +63,7 @@ jobs:
 
         echo "::endgroup::"
         echo "::group::Install TorchAudio"
-        conda install --quiet --yes pkg-config cmake>=3.18.0 ninja
+        conda install --quiet --yes cmake>=3.18.0 ninja
         pip3 install --progress-bar off -v -e . --no-use-pep517
 
         echo "::endgroup::"
@@ -75,7 +75,7 @@ jobs:
         conda install \
           --quiet --yes \
           -c conda-forge \
-          pandoc doxygen pysoundfile
+          sox libvorbis pandoc doxygen pysoundfile
         pip install --progress-bar off \
             git+https://github.com/kpu/kenlm/ flashlight-text \
             -r docs/requirements.txt -r docs/requirements-tutorials.txt

.github/workflows/integration-test.yml

@@ -24,7 +24,7 @@ jobs:
     - name: Install dependencies
       run: |
         sudo add-apt-repository -y ppa:jonathonf/ffmpeg-4
-        sudo apt install -y -qq pkg-config libavfilter-dev libavdevice-dev
+        sudo apt install -y -qq libavfilter-dev libavdevice-dev
     - name: Install packages
       run: |
         python -m pip install --quiet --upgrade pip

.github/workflows/unittest-linux-cpu.yml

@@ -50,6 +50,8 @@ jobs:
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_AUDIO_OUT_DEVICE=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MACOS=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_TEMPORARY_DISABLED=true
+        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_DECODER=true
+        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_ENCODER=true
         echo '::endgroup::'
 
         set -euxo pipefail

.github/workflows/unittest-linux-gpu.yml

@@ -38,6 +38,8 @@ jobs:
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_CUDA_SMALL_MEMORY=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_ON_PYTHON_310=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_TEMPORARY_DISABLED=true
+        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_DECODER=true
+        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_ENCODER=true
 
         # Set CHANNEL
         if [[(${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then
@@ -63,7 +65,7 @@ jobs:
 
         echo "::endgroup::"
         echo "::group::Install TorchAudio"
-        conda install --quiet --yes pkg-config 'cmake>=3.18.0' ninja
+        conda install --quiet --yes 'cmake>=3.18.0' ninja
         pip3 install --progress-bar off -v -e . --no-use-pep517
 
         echo "::endgroup::"
@@ -76,7 +78,7 @@ jobs:
           --quiet --yes \
           -c conda-forge \
           -c numba/label/dev \
-          'librosa==0.10.0' parameterized 'requests>=2.20'
+          sox libvorbis 'librosa==0.10.0' parameterized 'requests>=2.20'
         pip3 install --progress-bar off \
           kaldi-io \
           SoundFile \

.github/workflows/unittest-macos-cpu.yml

@@ -49,6 +49,8 @@ jobs:
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_sentencepiece=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_AUDIO_OUT_DEVICE=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_TEMPORARY_DISABLED=true
+        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_DECODER=true
+        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_ENCODER=true
         echo '::endgroup::'
 
         set -euxo pipefail

CONTRIBUTING.md

@@ -49,13 +49,11 @@ torchaudio.
 conda install pytorch -c pytorch-nightly
 ```
 
-### Install build dependencies
+### Install build/runtime dependencies
 
 ```bash
 # Install build-time dependencies
 pip install cmake ninja
-# [optional for sox]
-conda install pkg-config
 # [optional for ffmpeg]
 conda install ffmpeg
 ```

docs/source/build.jetson.rst

@@ -121,7 +121,7 @@ Verify the installation by checking the version and CUDA device accessibility.
 
 .. code-block::
 
-   pip install cmake ninja pkg-config
+   pip install cmake ninja
 
 2. Install dependencies
 ~~~~~~~~~~~~~~~~~~~~~~~

docs/source/build.linux.rst

@@ -22,7 +22,7 @@ Here, we install nightly build.
 
 .. code-block::
 
-   conda install cmake ninja pkg-config
+   conda install cmake ninja
 
 4. Clone the torchaudio repository
 ----------------------------------

docs/source/build.rst

@@ -10,7 +10,6 @@ TorchAudio integrates PyTorch for numerical computation and third party librarie
    - `GCC <https://gcc.gnu.org/>`_ (Linux)
    - `Clang <https://clang.llvm.org/>`_ (macOS)
    - `MSVC <https://visualstudio.microsoft.com>`_  2019 or newer (Windows)
-- `pkg-config <https://www.freedesktop.org/wiki/Software/pkg-config/>`_ (Linux/macOS, if building sox extension)
 - `CUDA toolkit <https://developer.nvidia.com/cudnn>`_ and `cuDNN <https://developer.nvidia.com/cudnn>`_ (if building CUDA extension)
 
 Most of the tools are available in `Conda <https://conda.io/>`_, so we recommend using conda.

packaging/torchaudio/meta.yaml

@@ -14,7 +14,6 @@ requirements:
   host:
     - python
     - setuptools
-    - pkg-config  # [not win]
     - cmake
     - ninja
     - pytorch-mutex 1.0 {{ build_variant }}  # [not osx ]

setup.py

@@ -4,7 +4,6 @@
 import re
 import shutil
 import subprocess
-import sys
 from pathlib import Path
 
 import torch
@@ -83,18 +82,6 @@ def _get_packages(branch_name, tag):
     return find_packages(exclude=exclude)
 
 
-def _init_submodule():
-    print(" --- Initializing submodules")
-    try:
-        subprocess.check_call(["git", "submodule", "init"])
-        subprocess.check_call(["git", "submodule", "update"])
-    except Exception:
-        print(" --- Submodule initalization failed")
-        print("Please run:\n\tgit submodule update --init --recursive")
-        sys.exit(1)
-    print(" --- Initialized submodule")
-
-
 def _parse_url(path):
     with open(path, "r") as file_:
         for line in file_:
@@ -104,32 +91,13 @@ def _parse_url(path):
                 yield url
 
 
-def _parse_sources():
-    third_party_dir = ROOT_DIR / "third_party"
-    libs = ["sox"]
-    archive_dir = third_party_dir / "archives"
-    archive_dir.mkdir(exist_ok=True)
-    for lib in libs:
-        cmake_file = third_party_dir / lib / "CMakeLists.txt"
-        for url in _parse_url(cmake_file):
-            path = archive_dir / os.path.basename(url)
-            yield path, url
-
-
 def _fetch_archives(src):
     for dest, url in src:
         if not dest.exists():
             print(f" --- Fetching {os.path.basename(dest)}")
             torch.hub.download_url_to_file(url, dest, progress=False)
 
 
-def _fetch_third_party_libraries():
-    # Revert this when a submodule is added again
-    # _init_submodule()
-    if os.name != "nt":
-        _fetch_archives(_parse_sources())
-
-
 def _main():
     sha = _run_cmd(["git", "rev-parse", "HEAD"])
     branch = _run_cmd(["git", "rev-parse", "--abbrev-ref", "HEAD"])
@@ -143,7 +111,6 @@ def _main():
     print("-- Building version", version)
 
     _make_version_file(version, sha)
-    _fetch_third_party_libraries()
 
     with open("README.md") as f:
         long_description = f.read()