diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5da60e13bd310..0eb68ef44511f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,270 +1,23 @@ -# Copyright (c) 2023 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - name: CI on: - # See: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request. pull_request: - # See: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#push. push: branches: - '**' tags-ignore: - '**' - -concurrency: - group: ${{ github.event_name != 'pull_request' && github.run_id || github.ref }} - cancel-in-progress: true - -env: - CI_FAILFAST_TEST_LEAVE_DANGLING: 1 # GHA does not care about dangling processes and setting this variable avoids killing the CI script itself on error - MAKEJOBS: '-j10' - jobs: - test-each-commit: - name: 'test each commit' - runs-on: ubuntu-24.04 - if: github.event_name == 'pull_request' && github.event.pull_request.commits != 1 - timeout-minutes: 360 # Use maximum time, see https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idtimeout-minutes. Assuming a worst case time of 1 hour per commit, this leads to a --max-count=6 below. - env: - MAX_COUNT: 6 + build-and-test: + runs-on: [self-hosted, linux, x64] + timeout-minutes: 20 steps: - - name: Determine fetch depth - run: echo "FETCH_DEPTH=$((${{ github.event.pull_request.commits }} + 2))" >> "$GITHUB_ENV" - - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: ${{ env.FETCH_DEPTH }} - - name: Determine commit range - run: | - # Checkout HEAD~ and find the test base commit - # Checkout HEAD~ because it would be wasteful to rerun tests on the PR - # head commit that are already run by other jobs. - git checkout HEAD~ - # Figure out test base commit by listing ancestors of HEAD, excluding - # ancestors of the most recent merge commit, limiting the list to the - # newest MAX_COUNT ancestors, ordering it from oldest to newest, and - # taking the first one. - # - # If the branch contains up to MAX_COUNT ancestor commits after the - # most recent merge commit, all of those commits will be tested. If it - # contains more, only the most recent MAX_COUNT commits will be - # tested. - # - # In the command below, the ^@ suffix is used to refer to all parents - # of the merge commit as described in: - # https://git-scm.com/docs/git-rev-parse#_other_rev_parent_shorthand_notations - # and the ^ prefix is used to exclude these parents and all their - # ancestors from the rev-list output as described in: - # https://git-scm.com/docs/git-rev-list - MERGE_BASE=$(git rev-list -n1 --merges HEAD) - EXCLUDE_MERGE_BASE_ANCESTORS= - # MERGE_BASE can be empty due to limited fetch-depth - if test -n "$MERGE_BASE"; then - EXCLUDE_MERGE_BASE_ANCESTORS=^${MERGE_BASE}^@ - fi - echo "TEST_BASE=$(git rev-list -n$((${{ env.MAX_COUNT }} + 1)) --reverse HEAD $EXCLUDE_MERGE_BASE_ANCESTORS | head -1)" >> "$GITHUB_ENV" - - run: | - sudo apt-get update - sudo apt-get install clang ccache build-essential cmake pkg-config python3-zmq libevent-dev libboost-dev libsqlite3-dev libdb++-dev systemtap-sdt-dev libzmq3-dev qtbase5-dev qttools5-dev qttools5-dev-tools qtwayland5 libqrencode-dev -y - - name: Compile and run tests - run: | - # Run tests on commits after the last merge commit and before the PR head commit - # Use clang++, because it is a bit faster and uses less memory than g++ - git rebase --exec "echo Running test-one-commit on \$( git log -1 ) && CC=clang CXX=clang++ cmake -B build -DWERROR=ON -DWITH_ZMQ=ON -DBUILD_GUI=ON -DBUILD_BENCH=ON -DBUILD_FUZZ_BINARY=ON -DWITH_BDB=ON -DWITH_USDT=ON && cmake --build build -j $(nproc) && ctest --output-on-failure --test-dir build -j $(nproc) && ./build/test/functional/test_runner.py -j $(( $(nproc) * 2 ))" ${{ env.TEST_BASE }} - - macos-native-arm64: - name: 'macOS 14 native, arm64, no depends, sqlite only, gui' - # Use latest image, but hardcode version to avoid silent upgrades (and breaks). - # See: https://github.com/actions/runner-images#available-images. - runs-on: macos-14 - - # No need to run on the read-only mirror, unless it is a PR. - if: github.repository != 'bitcoin-core/gui' || github.event_name == 'pull_request' - - timeout-minutes: 120 - - env: - DANGER_RUN_CI_ON_HOST: 1 - FILE_ENV: './ci/test/00_setup_env_mac_native.sh' - BASE_ROOT_DIR: ${{ github.workspace }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Clang version - run: | - sudo xcode-select --switch /Applications/Xcode_15.0.app - clang --version - - - name: Install Homebrew packages - env: - HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1 - run: | - # A workaround for "The `brew link` step did not complete successfully" error. - brew install --quiet python@3 || brew link --overwrite python@3 - brew install --quiet coreutils ninja pkg-config gnu-getopt ccache boost libevent zeromq qt@5 qrencode - - - name: Set Ccache directory - run: echo "CCACHE_DIR=${RUNNER_TEMP}/ccache_dir" >> "$GITHUB_ENV" - - - name: Restore Ccache cache - id: ccache-cache - uses: actions/cache/restore@v4 - with: - path: ${{ env.CCACHE_DIR }} - key: ${{ github.job }}-ccache-${{ github.run_id }} - restore-keys: ${{ github.job }}-ccache- - - - name: CI script - run: ./ci/test_run_all.sh - - - name: Save Ccache cache - uses: actions/cache/save@v4 - if: github.event_name != 'pull_request' && steps.ccache-cache.outputs.cache-hit != 'true' - with: - path: ${{ env.CCACHE_DIR }} - # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache - key: ${{ github.job }}-ccache-${{ github.run_id }} - - win64-native: - name: 'Win64 native, VS 2022' - # Use latest image, but hardcode version to avoid silent upgrades (and breaks). - # See: https://github.com/actions/runner-images#available-images. - runs-on: windows-2022 - - # No need to run on the read-only mirror, unless it is a PR. - if: github.repository != 'bitcoin-core/gui' || github.event_name == 'pull_request' - - env: - PYTHONUTF8: 1 - TEST_RUNNER_TIMEOUT_FACTOR: 40 - - steps: - - name: Checkout + - name: Checkout repo uses: actions/checkout@v4 - - - name: Configure Developer Command Prompt for Microsoft Visual C++ - # Using microsoft/setup-msbuild is not enough. - uses: ilammy/msvc-dev-cmd@v1 - with: - arch: x64 - - - name: Get tool information - run: | - cmake -version | Tee-Object -FilePath "cmake_version" - Write-Output "---" - msbuild -version | Tee-Object -FilePath "msbuild_version" - $env:VCToolsVersion | Tee-Object -FilePath "toolset_version" - py -3 --version - Write-Host "PowerShell version $($PSVersionTable.PSVersion.ToString())" - - - name: Using vcpkg with MSBuild - run: | - Set-Location "$env:VCPKG_INSTALLATION_ROOT" - Add-Content -Path "triplets\x64-windows.cmake" -Value "set(VCPKG_BUILD_TYPE release)" - Add-Content -Path "triplets\x64-windows-static.cmake" -Value "set(VCPKG_BUILD_TYPE release)" - - - name: vcpkg tools cache - uses: actions/cache@v4 - with: - path: C:/vcpkg/downloads/tools - key: ${{ github.job }}-vcpkg-tools - - - name: Restore vcpkg binary cache - uses: actions/cache/restore@v4 - id: vcpkg-binary-cache + - uses: cachix/install-nix-action@v27 with: - path: ~/AppData/Local/vcpkg/archives - key: ${{ github.job }}-vcpkg-binary-${{ hashFiles('cmake_version', 'msbuild_version', 'toolset_version', 'vcpkg.json') }} - - - name: Generate build system - run: | - cmake -B build --preset vs2022-static -DCMAKE_TOOLCHAIN_FILE="$env:VCPKG_INSTALLATION_ROOT\scripts\buildsystems\vcpkg.cmake" -DBUILD_GUI=ON -DWITH_BDB=ON -DWITH_ZMQ=ON -DBUILD_BENCH=ON -DBUILD_FUZZ_BINARY=ON -DWERROR=ON - - - name: Save vcpkg binary cache - uses: actions/cache/save@v4 - if: github.event_name != 'pull_request' && steps.vcpkg-binary-cache.outputs.cache-hit != 'true' - with: - path: ~/AppData/Local/vcpkg/archives - key: ${{ github.job }}-vcpkg-binary-${{ hashFiles('cmake_version', 'msbuild_version', 'toolset_version', 'vcpkg.json') }} - - - name: Build - working-directory: build - run: | - cmake --build . -j $env:NUMBER_OF_PROCESSORS --config Release - - - name: Run test suite - working-directory: build - run: | - ctest --output-on-failure -j $env:NUMBER_OF_PROCESSORS -C Release - - - name: Run functional tests - working-directory: build + nix_path: nixpkgs=channel:nixos-unstable + - name: Build & test release env: - BITCOIND: '${{ github.workspace }}\build\src\Release\bitcoind.exe' - BITCOINCLI: '${{ github.workspace }}\build\src\Release\bitcoin-cli.exe' - BITCOINUTIL: '${{ github.workspace }}\build\src\Release\bitcoin-util.exe' - BITCOINWALLET: '${{ github.workspace }}\build\src\Release\bitcoin-wallet.exe' - TEST_RUNNER_EXTRA: ${{ github.event_name != 'pull_request' && '--extended' || '' }} - shell: cmd - run: py -3 test\functional\test_runner.py --jobs %NUMBER_OF_PROCESSORS% --ci --quiet --tmpdirprefix=%RUNNER_TEMP% --combinedlogslen=99999999 --timeout-factor=%TEST_RUNNER_TIMEOUT_FACTOR% %TEST_RUNNER_EXTRA% - - - name: Clone fuzz corpus + NIX_PATH: nixpkgs=channel:nixos-unstable run: | - git clone --depth=1 https://github.com/bitcoin-core/qa-assets "$env:RUNNER_TEMP\qa-assets" - Set-Location "$env:RUNNER_TEMP\qa-assets" - Write-Host "Using qa-assets repo from commit ..." - git log -1 - - - name: Run fuzz binaries - working-directory: build - env: - BITCOINFUZZ: '${{ github.workspace }}\build\src\test\fuzz\Release\fuzz.exe' - shell: cmd - run: py -3 test\fuzz\test_runner.py --par %NUMBER_OF_PROCESSORS% --loglevel DEBUG %RUNNER_TEMP%\qa-assets\fuzz_corpora - - asan-lsan-ubsan-integer-no-depends-usdt: - name: 'ASan + LSan + UBSan + integer, no depends, USDT' - runs-on: ubuntu-24.04 # has to match container in ci/test/00_setup_env_native_asan.sh for tracing tools - # No need to run on the read-only mirror, unless it is a PR. - if: github.repository != 'bitcoin-core/gui' || github.event_name == 'pull_request' - timeout-minutes: 120 - env: - FILE_ENV: "./ci/test/00_setup_env_native_asan.sh" - DANGER_CI_ON_HOST_CACHE_FOLDERS: 1 - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set Ccache directory - run: echo "CCACHE_DIR=${RUNNER_TEMP}/ccache_dir" >> "$GITHUB_ENV" - - - name: Set base root directory - run: echo "BASE_ROOT_DIR=${RUNNER_TEMP}" >> "$GITHUB_ENV" - - - name: Restore Ccache cache - id: ccache-cache - uses: actions/cache/restore@v4 - with: - path: ${{ env.CCACHE_DIR }} - key: ${{ github.job }}-ccache-${{ github.run_id }} - restore-keys: ${{ github.job }}-ccache- - - - name: Enable bpfcc script - # In the image build step, no external environment variables are available, - # so any settings will need to be written to the settings env file: - run: sed -i "s|\${INSTALL_BCC_TRACING_TOOLS}|true|g" ./ci/test/00_setup_env_native_asan.sh - - - name: CI script - run: ./ci/test_run_all.sh - - - name: Save Ccache cache - uses: actions/cache/save@v4 - if: github.event_name != 'pull_request' && steps.ccache-cache.outputs.cache-hit != 'true' - with: - path: ${{ env.CCACHE_DIR }} - # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache - key: ${{ github.job }}-ccache-${{ github.run_id }} + nix-shell --command "just run-ci" diff --git a/justfile b/justfile new file mode 100644 index 0000000000000..0feef8a422371 --- /dev/null +++ b/justfile @@ -0,0 +1,67 @@ +set shell := ["bash", "-uc"] + +os := os() + +default: + just --list + +# Build default project +[group('build')] +build *args: clean + cmake -B build {{ args }} + cmake --build build -j {{ num_cpus() }} + +# Build with all optional modules +[group('build')] +build-dev *args: clean + cmake -B build --preset dev-mode {{ args }} + cmake --build build -j {{ num_cpus() }} + +# Build for the CI, including bench_bitcoin +[private] +[group('ci')] +build-ci: clean + cmake -B build -DBUILD_BENCH=ON + cmake --build build -j {{ num_cpus() }} +# Re-build current config +[group('build')] +rebuild: + cmake --build build -j {{ num_cpus() }} + +# Clean build dir using git clean -dfx +[group('build')] +clean: + git clean -dfx + +# Run unit tests +[group('test')] +test-unit: + ctest --test-dir build -j {{ num_cpus() }} + +# Run all functional tests +[group('test')] +test-func: + build/test/functional/test_runner.py -j {{ num_cpus() }} + +# Run all unit and functional tests +[group('test')] +test: test-unit test-func + +# Run a single functional test (filename.py) +[group('test')] +test-func1 test: + build/test/functional/test_runner.py {{ test }} + +# Run a single unit test suite +[group('test')] +test-unit1 suite: + build/src/test/test_bitcoin --log_level=all --run_test={{ suite }} + +# Run benchmarks +[group('perf')] +bench: + build/src/bench/bench_bitcoin + +# Run the CI workflow +[group('ci')] +run-ci: build-ci bench test diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000000000..6cf45818c220a --- /dev/null +++ b/shell.nix @@ -0,0 +1,125 @@ +# Copyright 0xB10C + +{ pkgs ? import (fetchTarball "https://github.com/nixos/nixpkgs/archive/nixos-unstable.tar.gz") {}, + bdbVersion ? "", + spareCores ? 0, + withClang ? false, + withDebug ? false, + withGui ? false, +}: +let + inherit (pkgs.lib) optionals strings; + binDirs = + [ "\$PWD/src" ] + ++ optionals withGui [ "\$PWD/src/qt" ]; + configureFlags = + [ "--with-boost-libdir=$NIX_BOOST_LIB_DIR" ] + ++ optionals ((builtins.elem bdbVersion ["" "db48" "db5"]) || abort "Unsupported bdbVersion value: ${bdbVersion}") [] + ++ optionals (bdbVersion == "") [ "--without-bdb" ] + ++ optionals (!(builtins.elem bdbVersion ["" "db48"])) [ "--with-incompatible-bdb" ] + ++ optionals withClang [ "CXX=clang++" "CC=clang" ] + ++ optionals withDebug [ "--enable-debug" ] + ++ optionals withGui [ + "--with-gui=qt5" + "--with-qt-bindir=${pkgs.qt5.qtbase.dev}/bin:${pkgs.qt5.qttools.dev}/bin" + ]; + jobs = + if (strings.hasSuffix "linux" builtins.currentSystem) then "$(($(nproc)-${toString spareCores}))" + else if (strings.hasSuffix "darwin" builtins.currentSystem) then "$(($(sysctl -n hw.physicalcpu)-${toString spareCores}))" + else "6"; +in pkgs.mkShell { + nativeBuildInputs = with pkgs; [ + autoconf + automake + libtool + pkg-config + boost + libevent + zeromq + sqlite + clang_18 + + # tests + hexdump + + # compiler output caching per + # https://github.com/bitcoin/bitcoin/blob/master/doc/productivity.md#cache-compilations-with-ccache + ccache + + # generating compile_commands.json for clang-format, clang-tidy, LSPs etc + # https://github.com/bitcoin/bitcoin/blob/master/doc/developer-notes.md#running-clang-tidy + # $ a && c && m clean && bear --config src/.bear-tidy-config -- make -j6 + clang-tools_18 + bear + + # for newer cmake building + cmake + + # depends + byacc + + # functional tests & linting + python3 + python3Packages.flake8 + python3Packages.lief + python3Packages.autopep8 + python3Packages.mypy + python3Packages.requests + python3Packages.pyzmq + + # benchmarking + python3Packages.pyperf + + # debugging + gdb + + # tracing + libsystemtap + linuxPackages.bpftrace + linuxPackages.bcc + + ] + ++ lib.optionals (bdbVersion == "db48") [ + db48 + ] + ++ lib.optionals (bdbVersion == "db5") [ + db5 + ] + ++ lib.optionals withGui [ + # bitcoin-qt + qt5.qtbase + # required for bitcoin-qt for "LRELEASE" etc + qt5.qttools + ]; + buildInputs = with pkgs; [ + just + bash + ]; + + + + # Modifies the Nix clang++ wrapper to avoid warning: + # "_FORTIFY_SOURCE requires compiling with optimization (-O)" + hardeningDisable = if withDebug then [ "all" ] else [ ]; + + # Fixes xcb plugin error when trying to launch bitcoin-qt + QT_QPA_PLATFORM_PLUGIN_PATH = if withGui then "${pkgs.qt5.qtbase.bin}/lib/qt-${pkgs.qt5.qtbase.version}/plugins/platforms" else ""; + + shellHook = '' + echo "Bitcoin Core build nix-shell" + echo "" + + BCC_EGG=${pkgs.linuxPackages.bcc}/${pkgs.python3.sitePackages}/bcc-${pkgs.linuxPackages.bcc.version}-py3.${pkgs.python3.sourceVersion.minor}.egg + + echo "adding bcc egg to PYTHONPATH: $BCC_EGG" + if [ -f $BCC_EGG ]; then + export PYTHONPATH="$PYTHONPATH:$BCC_EGG" + echo "" + else + echo "The bcc egg $BCC_EGG does not exist. Maybe the python or bcc version is different?" + fi + + echo "adding ${builtins.concatStringsSep ":" binDirs} to \$PATH to make running built binaries more natural" + export PATH=$PATH:${builtins.concatStringsSep ":" binDirs}; + ''; +}