Add requirement.txt for ONNX's dependency (onnx#3448)

* fix ort failure and add manually trigger Signed-off-by: Chun-Wei Chen <[email protected]> * fix syntax error Signed-off-by: Chun-Wei Chen <[email protected]> * add master trigger first Signed-off-by: Chun-Wei Chen <[email protected]> * pip install flatbuffers Signed-off-by: Chun-Wei Chen <[email protected]> * apply it to all CIs Signed-off-by: Chun-Wei Chen <[email protected]> * add requirement Signed-off-by: Chun-Wei Chen <[email protected]> * use requirement for install_require Signed-off-by: Chun-Wei Chen <[email protected]> * add six and typin-ext in .dev Signed-off-by: Chun-Wei Chen <[email protected]> * reuse requirement-release.txt Signed-off-by: Chun-Wei Chen <[email protected]> * update requreiment-release.txt Signed-off-by: Chun-Wei Chen <[email protected]> * install protobuf 3.11.3 after installing protobuf-compiler Signed-off-by: Chun-Wei Chen <[email protected]> * install protobuf 3.11.3 rb Signed-off-by: Chun-Wei Chen <[email protected]> * install protobuf without brew Signed-off-by: Chun-Wei Chen <[email protected]> * remove conda install in Mac-CI Signed-off-by: Chun-Wei Chen <[email protected]> * do not use requirements.txt in i686 Signed-off-by: Chun-Wei Chen <[email protected]> * build 3.11.3 protobuf from source Signed-off-by: Chun-Wei Chen <[email protected]> * ipython==7.16.1 Signed-off-by: Chun-Wei Chen <[email protected]> * use latest numpy to prevent timeout Signed-off-by: Chun-Wei Chen <[email protected]> * bump protobuf to 3.16.0 Signed-off-by: Chun-Wei Chen <[email protected]> * remove all 3.11.3 Signed-off-by: Chun-Wei Chen <[email protected]> * remove minimum requirement for protobuf Signed-off-by: Chun-Wei Chen <[email protected]>
dyastremsky · Jul 20, 2021 · 27be295 · 27be295
1 parent 2875f51
commit 27be295
Show file tree

Hide file tree

Showing 13 changed files with 111 additions and 89 deletions.
diff --git a/.azure-pipelines/Linux-CI.yml b/.azure-pipelines/Linux-CI.yml
@@ -41,7 +41,7 @@ jobs:
       addToPath: true
 
   - script: |
-      python -m pip install virtualenv
+      python -m pip -q install virtualenv
       python -m virtualenv py$(python.version)
       source py$(python.version)/bin/activate
 
@@ -56,8 +56,8 @@ jobs:
       sudo make install
       cd ../..
 
-      python -m pip install --upgrade pip
-      python -m pip install numpy protobuf==3.16.0
+      python -m pip install -q --upgrade pip
+      python -m pip install -q -r requirements-release.txt
 
       sudo apt-get install -qq -o=Dpkg::Use-Pty=0 -y --no-install-recommends dos2unix
       git submodule update --init --recursive
@@ -80,7 +80,7 @@ jobs:
       source py$(python.version)/bin/activate
 
       # lint python code
-      python -m pip install --quiet flake8
+      python -m pip install -q flake8
       flake8 onnx
       if [ $? -ne 0 ]; then
         echo "flake8 returned failures"
@@ -111,12 +111,6 @@ jobs:
       ! grep -R --include='*.cc' --include='*.h' 'namespace onnx' .
       ! grep -R --include='*.cc' --include='*.h' 'onnx::' .
 
-      # onnx python api tests
-      # pytest 6.0 made deprecation warnings fail by default, pinning pytest to 5.4.3.
-      # TODO replace deprecated function with the suggested one. https://docs.pytest.org/en/stable/deprecations.html#id5
-      # TODO Remove fixed ipython 7.16.1 once ONNX has removed Python 3.6
-      python -m pip install --quiet pytest==5.4.3 nbval ipython==7.16.1
-
       pytest
       if [ $? -ne 0 ]; then
         echo "pytest failed"
@@ -139,14 +133,14 @@ jobs:
 
       # Mypy only works with our generated _pb.py files when we install in develop mode, so let's do that
       python -m pip uninstall -y onnx
-      python -m pip install --no-use-pep517 -e .[mypy]
+      python -m pip install -q --no-use-pep517 -e .[mypy]
       python setup.py --quiet typecheck
       if [ $? -ne 0 ]; then
         echo "type check failed"
         exit 1
       fi
       python -m pip uninstall -y onnx
       rm -rf .setuptools-cmake-build
-      python -m pip install .
+      python -m pip install -q .
 
     displayName: 'Run ONNX tests'
diff --git a/.azure-pipelines/MacOS-CI.yml b/.azure-pipelines/MacOS-CI.yml
@@ -30,12 +30,26 @@ jobs:
       versionSpec: '$(python.version)'
 
   - script: |
-      git submodule update --init --recursive
-      python -m pip install --upgrade setuptools
-      python -m pip install numpy
-      conda install -y -c conda-forge pybind11 protobuf
+      # Install protobuf 3.16.0 from source
+      export NUM_CORES=`sysctl -n hw.ncpu`
+      echo Using $NUM_CORES cores
       brew update
-      brew install protobuf
+      brew install autoconf && brew install automake
+      export ONNX_PATH=$(pwd)
+      cd ..
+      wget https://github.com/protocolbuffers/protobuf/releases/download/v3.16.0/protobuf-cpp-3.16.0.tar.gz
+      tar -xvf protobuf-cpp-3.16.0.tar.gz
+      cd protobuf-3.16.0
+      mkdir build_source && cd build_source
+      cmake ../cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release
+      make -j${NUM_CORES}
+      make install
+      export PATH=$(pwd)/bin:$PATH
+      cd $ONNX_PATH
+
+      git submodule update --init --recursive
+      python -m pip install -q --upgrade pip
+      python -m pip install -q -r requirements-release.txt
       export ONNX_BUILD_TESTS=1
       if [ '$(onnx_debug)' == '1' ]; then
         export DEBUG=1
@@ -48,12 +62,12 @@ jobs:
         export CMAKE_ARGS="${CMAKE_ARGS} -DONNX_USE_LITE_PROTO=ON"
       fi
       export ONNX_NAMESPACE=ONNX_NAMESPACE_FOO_BAR_FOR_CI
-      python setup.py --quiet install
+      python setup.py -q install
     displayName: 'Install dependencies and ONNX'
 
   - script: |
       # lint python code
-      pip install --quiet flake8
+      python -m pip install -q flake8
       flake8
       if [ $? -ne 0 ]; then
         echo "flake8 returned failures"
@@ -79,12 +93,6 @@ jobs:
       ! grep -R --include='*.cc' --include='*.h' 'namespace onnx' .
       ! grep -R --include='*.cc' --include='*.h' 'onnx::' .
 
-      # onnx python api tests
-      # pytest 6.0 made deprecation warnings fail by default, pinning pytest to 5.4.3.
-      # TODO replace deprecated function with the suggested one. https://docs.pytest.org/en/stable/deprecations.html#id5
-      # TODO Remove fixed ipython 7.16.1 once ONNX has removed Python 3.6
-      pip install --quiet pytest==5.4.3 nbval ipython==7.16.1
-
       pytest
       if [ $? -ne 0 ]; then
         echo "pytest failed"

diff --git a/.azure-pipelines/Windows-CI.yml b/.azure-pipelines/Windows-CI.yml
@@ -42,11 +42,8 @@ jobs:
 
   - script: |
       call activate py$(python.version)
-      python -m pip install --upgrade pip
-      # pytest 6.0 made deprecation warnings fail by default, pinning pytest to 5.4.3.
-      # TODO replace deprecated function with the suggested one. https://docs.pytest.org/en/stable/deprecations.html#id5
-      # TODO Remove fixed ipython 7.16.1 once ONNX has removed Python 3.6
-      python -m pip install --quiet pytest==5.4.3 nbval numpy ipython==7.16.1
+      python -m pip install -q --upgrade pip
+      python -m pip install -q -r requirements-release.txt
 
       git submodule update --init --recursive
       set ONNX_BUILD_TESTS=1
@@ -55,7 +52,7 @@ jobs:
       set USE_MSVC_STATIC_RUNTIME=0
       set CMAKE_ARGS=-DONNX_USE_PROTOBUF_SHARED_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DONNX_WERROR=ON
 
-      python setup.py --quiet install
+      python setup.py -q install
       pytest
       IF NOT %ERRORLEVEL% EQU 0 (
         @echo "pytest failed"
@@ -73,8 +70,8 @@ jobs:
       )
 
       rm -rf .setuptools-cmake-build
-      pip install --quiet -e .[mypy]
-      python setup.py typecheck
+      python -m pip install -q -e .[mypy]
+      python setup.py -q typecheck
 
 
     displayName: Install and test ONNX
diff --git a/.github/workflows/manylinux/entrypoint.sh b/.github/workflows/manylinux/entrypoint.sh
@@ -6,7 +6,6 @@ set -e -x
 PY_VERSION=$1
 PLAT=$2
 GITHUB_EVENT_NAME=$3
-BUILD_REQUIREMENTS='numpy==1.16.6 protobuf==3.16.0'
 SYSTEM_PACKAGES='cmake3'
 if [ `uname -m` == 'aarch64' ]; then
  SYSTEM_PACKAGES='cmake'
@@ -52,11 +51,14 @@ export ONNX_ML=1
 export CMAKE_ARGS="-DPYTHON_INCLUDE_DIR=/opt/python/${PY_VER}/include/python${python_include[$PY_VERSION]}"
 
 # Update pip
-$PIP_COMMAND --upgrade --no-cache-dir pip
+$PIP_COMMAND --upgrade pip
 
-# Check if requirements were passed
-if [ ! -z "$BUILD_REQUIREMENTS" ]; then
-    $PIP_COMMAND --no-cache-dir ${BUILD_REQUIREMENTS} || { echo "Installing requirements failed."; exit 1; }
+# Install Python dependency
+if [ "$PLAT" == "manylinux2010_i686" ]; then
+    # pip install -r requirements-release will bump into issue in i686 due to pip install cryptography failure
+    $PIP_COMMAND numpy==1.16.6 protobuf==3.16.0 || { echo "Installing Python requirements failed."; exit 1; }
+else
+    $PIP_COMMAND -r requirements-release.txt || { echo "Installing Python requirements failed."; exit 1; }
 fi
 
 # Build wheels

diff --git a/.github/workflows/manylinux/test_package_i686.sh b/.github/workflows/manylinux/test_package_i686.sh
@@ -12,18 +12,18 @@ PY_VER=${python_map[$PY_VERSION]}
 yum install -y protobuf-devel cmake3
 
 PYTHON_BIN="/opt/python/${PY_VER}/bin/"
-PIP_INTALL_COMMAND="${PYTHON_BIN}pip install --no-cache-dir"
-PIP_UNINTALL_COMMAND="${PYTHON_BIN}pip uninstall -y"
+PIP_INSTALL_COMMAND="${PYTHON_BIN}pip install --no-cache-dir -q"
+PIP_UNINSTALL_COMMAND="${PYTHON_BIN}pip uninstall -y"
 PYTHON_COMAND="${PYTHON_BIN}python"
 PYTEST_COMMAND="${PYTHON_BIN}pytest"
 
-$PIP_INTALL_COMMAND --upgrade pip
-$PIP_INTALL_COMMAND numpy protobuf==3.16.0
-$PIP_INTALL_COMMAND dist/*manylinux2010_i686.whl
+$PIP_INSTALL_COMMAND --upgrade pip
+
+# pip install -r requirements-release will bump into issue in i686 due to pip install cryptography failure
+$PIP_INSTALL_COMMAND numpy protobuf==3.16.0 pytest==5.4.3 nbval ipython==7.16.1 || { echo "Installing Python requirements failed."; exit 1; }
+$PIP_INSTALL_COMMAND dist/*manylinux2010_i686.whl
 
 # pytest with the built wheel
-# TODO Remove fixed ipython 7.16.1 once ONNX has removed Python 3.6
-$PIP_INTALL_COMMAND pytest==5.4.3 nbval ipython==7.16.1
 $PYTEST_COMMAND
 
 # Test backend test data
@@ -34,13 +34,13 @@ $PYTHON_COMAND onnx/backend/test/cmd_tools.py generate-data
 $PYTHON_COMAND workflow_scripts/test_generated_backend.py
 
 # Verify ONNX with the latest numpy
-$PIP_UNINTALL_COMMAND numpy onnx && $PIP_INTALL_COMMAND numpy
-$PIP_INTALL_COMMAND dist/*manylinux2010_i686.whl
+$PIP_UNINSTALL_COMMAND numpy onnx && $PIP_INSTALL_COMMAND numpy
+$PIP_INSTALL_COMMAND dist/*manylinux2010_i686.whl
 $PYTEST_COMMAND
 
 # Verify ONNX with the latest protobuf
-$PIP_UNINTALL_COMMAND protobuf onnx && $PIP_INTALL_COMMAND protobuf
-$PIP_INTALL_COMMAND dist/*manylinux2010_i686.whl
+$PIP_UNINSTALL_COMMAND protobuf onnx && $PIP_INSTALL_COMMAND protobuf
+$PIP_INSTALL_COMMAND dist/*manylinux2010_i686.whl
 $PYTEST_COMMAND
 
 echo "Succesfully test the wheel"
diff --git a/.github/workflows/release_linux_x86_64.yml b/.github/workflows/release_linux_x86_64.yml
@@ -36,8 +36,8 @@ jobs:
 
     - name: Install python dependencies
       run: |
-        python -m pip install --upgrade pip
-        python -m pip install numpy protobuf==3.16.0
+        python -m pip install -q --upgrade pip
+        python -m pip install -q -r requirements-release.txt
 
     - name: Build manylinux2010_x86_64
       uses: docker://quay.io/pypa/manylinux2010_x86_64
@@ -50,7 +50,7 @@ jobs:
         cd ..
         git clone https://github.com/protocolbuffers/protobuf.git
         cd protobuf
-        git checkout v3.11.3
+        git checkout v3.16.0
         git submodule update --init --recursive
         mkdir build_source && cd build_source
 
@@ -61,8 +61,6 @@ jobs:
     - name: Test wheel with Python ${{ matrix.python-version }}
       run: |
         python -m pip install dist/*manylinux2010_x86_64.whl
-        # TODO Remove fixed ipython 7.16.1 once ONNX has removed Python 3.6
-        python -m pip install pytest==5.4.3 nbval ipython==7.16.1
         pytest
 
     - name: Test backend test data
@@ -81,7 +79,6 @@ jobs:
     - name: Upload wheel to TestPyPI weekly
       if: (github.event_name == 'schedule') # Only triggered by weekly event
       run: |
-        python -m pip install -q twine
         twine upload --verbose dist/*.whl --repository-url https://test.pypi.org/legacy/ -u ${{ secrets.TESTPYPI_USERNAME }} -p ${{ secrets.TESTPYPI_PASSWORD }}
 
     - name: Verify ONNX with the latest numpy
@@ -101,6 +98,6 @@ jobs:
     - name: Verify ONNX with ort-nightly
       if: ${{ always() }}
       run: |
-        python -m pip install flatbuffers
-        python -m pip install -i https://test.pypi.org/simple/ ort-nightly
+        python -m pip install -q flatbuffers
+        python -m pip install -q -i https://test.pypi.org/simple/ ort-nightly
         python onnx/test/test_with_ort.py
diff --git a/.github/workflows/release_mac.yml b/.github/workflows/release_mac.yml
@@ -38,11 +38,6 @@ jobs:
         python-version: ${{ matrix.python-version }}
         architecture: ${{ matrix.architecture }}
 
-    - name: Install Python dependencies
-      run: |
-        python -m pip install -q --upgrade pip
-        python -m pip install -q numpy==1.16.6 setuptools wheel
-
     - name: Install protobuf dependencies
       run: |
         # Install protobuf 3.16.0
@@ -62,6 +57,11 @@ jobs:
         export PATH=$(pwd)/bin:$PATH
         cd $ONNX_PATH
 
+    - name: Install Python dependencies
+      run: |
+        python -m pip install -q --upgrade pip
+        python -m pip install -q -r requirements-release.txt
+
     - name: Build wheel and install
       env:
         CC: "clang"
@@ -79,8 +79,6 @@ jobs:
 
     - name: Test the installed wheel
       run: |
-        # TODO Remove fixed ipython 7.16.1 once ONNX has removed Python 3.6
-        python -m pip install -q pytest==5.4.3 nbval ipython==7.16.1
         pytest
 
     - name: Test backend test data
@@ -99,7 +97,6 @@ jobs:
     - name: Upload wheel to TestPyPI weekly
       if: (github.event_name == 'schedule') # Only triggered by weekly event
       run: |
-        python -m pip install -q twine
         twine upload --verbose dist/*.whl --repository-url https://test.pypi.org/legacy/ -u ${{ secrets.TESTPYPI_USERNAME }} -p ${{ secrets.TESTPYPI_PASSWORD }}
 
     - name: Verify ONNX with the latest numpy
@@ -119,6 +116,6 @@ jobs:
     - name: Verify ONNX with ort-nightly
       if: ${{ always() }}
       run: |
-        python -m pip install flatbuffers
-        python -m pip install -i https://test.pypi.org/simple/ ort-nightly
+        python -m pip install -q flatbuffers
+        python -m pip install -q -i https://test.pypi.org/simple/ ort-nightly
         python onnx/test/test_with_ort.py
diff --git a/.github/workflows/release_win.yml b/.github/workflows/release_win.yml
@@ -58,9 +58,10 @@ jobs:
 
     - name: Install python dependencies
       run: |
-        python -m pip install --upgrade pip
-        python -m pip install numpy==1.16.6 wheel
-            
+        python -m pip install -q --upgrade pip
+        cd onnx
+        python -m pip install -q -r requirements-release.txt
+
     - name: Build ONNX wheel
       run: |        
         $arch = 'x64'
@@ -101,12 +102,9 @@ jobs:
         }
         Get-ChildItem -Path dist/*.whl | foreach {python -m pip install --upgrade $_.fullname} 
   
-
     - name: Test the installed wheel
       run: |
         cd onnx
-        # TODO Remove fixed ipython 7.16.1 once ONNX has removed Python 3.6
-        python -m pip install -q pytest==5.4.3 nbval ipython==7.16.1
         pytest
 
     - name: Test the uploaded backend test data
@@ -126,7 +124,6 @@ jobs:
     - name: Upload wheel to TestPyPI weekly
       if: (github.event_name == 'schedule') # Only triggered by weekly event
       run: |
-        python -m pip install -q twine
         twine upload --verbose onnx/dist/*.whl --repository-url https://test.pypi.org/legacy/ -u ${{ secrets.TESTPYPI_USERNAME }} -p ${{ secrets.TESTPYPI_PASSWORD }}
 
     - name: Verify ONNX with the latest numpy
@@ -150,7 +147,7 @@ jobs:
       run: |
         # ort-nightly does not support x86
         if ('${{ matrix.architecture }}' -eq 'x64') {
-          python -m pip install flatbuffers
-          python -m pip install -i https://test.pypi.org/simple/ ort-nightly
+          python -m pip install -q flatbuffers
+          python -m pip install -q -i https://test.pypi.org/simple/ ort-nightly
           python onnx\onnx\test\test_with_ort.py
         }