From 05690b64901ff5f007dd30ee61a41fcd6ea5fafd Mon Sep 17 00:00:00 2001 From: Quentin Deslandes Date: Fri, 1 Nov 2024 08:59:04 +0100 Subject: [PATCH] build: refactor CI --- .github/workflows/ci.yaml | 115 +++++++-------------------------- .github/workflows/fork.yaml | 42 ------------ .github/workflows/pages.yaml | 51 --------------- src/bpfilter/cgen/program.h | 12 ++-- src/bpfilter/cgen/stub.c | 5 ++ tests/benchmark/CMakeLists.txt | 1 - tests/benchmark/benchmark.cpp | 6 +- tests/e2e/CMakeLists.txt | 3 +- tests/harness/daemon.c | 9 ++- tools/ci.yaml | 84 ++++++++++++++++++++++++ tools/fedora-39.Dockerfile | 26 ++++++++ tools/fedora-40.Dockerfile | 26 ++++++++ tools/fedora-41.Dockerfile | 26 ++++++++ tools/ubuntu-24.04.Dockerfile | 32 +++++++++ tools/ubuntu-24.10.Dockerfile | 32 +++++++++ 15 files changed, 272 insertions(+), 198 deletions(-) delete mode 100644 .github/workflows/fork.yaml delete mode 100644 .github/workflows/pages.yaml create mode 100644 tools/ci.yaml create mode 100644 tools/fedora-39.Dockerfile create mode 100644 tools/fedora-40.Dockerfile create mode 100644 tools/fedora-41.Dockerfile create mode 100644 tools/ubuntu-24.04.Dockerfile create mode 100644 tools/ubuntu-24.10.Dockerfile diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0b7648f7..17f0f72c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,103 +1,32 @@ -name: CI +name: Testing on CI on: push: branches: - - main - pull_request: - branches: - - main + - "**" + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} permissions: - checks: write - pull-requests: write + contents: read + packages: write jobs: - full: - if: github.repository == 'facebook/bpfilter' - strategy: - fail-fast: false - matrix: - system: ["fedora:41"] - arch: ["X64", "ARM64"] - mode: ["release", "debug"] - container: "${{ matrix.system }}" - runs-on: [self-hosted, "${{ matrix.arch }}"] - steps: - - name: Check out repository code - uses: actions/checkout@v4 - - name: Install dependencies (Fedora) - run: | - sudo dnf --disablerepo=* --enablerepo=fedora,updates --setopt=install_weak_deps=False -y install \ - bison bpftool clang clang-tools-extra cmake doxygen flex g++ git gcc jq lcov libasan libbpf-devel \ - libcmocka-devel libnl3-devel libubsan pkgconf python3-breathe python3-furo python3-linuxdoc \ - python3-sphinx - - name: Configure build - run: cmake -S $GITHUB_WORKSPACE -B $GITHUB_WORKSPACE/build -DCMAKE_BUILD_TYPE=${{ matrix.mode }} - - name: Build - run: make -C $GITHUB_WORKSPACE/build - - name: Run unit tests - run: make -C $GITHUB_WORKSPACE/build test - - name: Coverage - run: make -C $GITHUB_WORKSPACE/build coverage - - name: Upload coverage report to Codecov - uses: codecov/codecov-action@v3 - with: - files: build/doc/lcov.out - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - - name: Check style - run: make -C $GITHUB_WORKSPACE/build check - - name: Generate documentation - run: make -C $GITHUB_WORKSPACE/build doc - - build: - if: github.repository == 'facebook/bpfilter' - strategy: - fail-fast: false - matrix: - image: ["fedora:39", "fedora:40", "ubuntu:24.04"] - arch: ["X64", "ARM64"] - container: ${{ matrix.image }} - runs-on: [self-hosted, "${{ matrix.arch }}"] + tests: + runs-on: ["4-core-ubuntu-arm"] + container: + image: ghcr.io/facebook/bpfilter:fedora-41-arm64 + options: --privileged --pid=host -v /home/quentin/Projects/bpfilter:/bpfilter steps: - - name: Check out repository code - uses: actions/checkout@v4 - - name: Install dependencies (Fedora) - if: matrix.image == 'fedora:39' || matrix.image == 'fedora:40' - run: | - sudo dnf --disablerepo=* --enablerepo=fedora,updates --setopt=install_weak_deps=False -y install \ - bison bpftool clang clang-tools-extra cmake doxygen flex g++ git gcc jq lcov libasan libbpf-devel \ - libcmocka-devel libnl3-devel libubsan pkgconf python3-breathe python3-furo python3-linuxdoc \ - python3-sphinx - - name: Install dependencies (Ubuntu) - if: matrix.image == 'ubuntu:24.04' - run: | - apt-get update - DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install \ - bison \ - clang \ - clang-format \ - clang-tidy \ - cmake \ - doxygen \ - flex \ - furo \ - git \ - jq \ - lcov \ - libbpf-dev \ - libcmocka-dev \ - libnl-3-dev \ - linux-tools-common \ - python3-breathe \ - python3-pip \ - python3-sphinx \ - pkgconf && \ - pip3 install --break-system-packages linuxdoc + - name: Checkout repository + uses: actions/checkout@v2 + - name: Mount bpffs + run: mount bpffs -t bpf /sys/fs/bpf + - name: Check + run: uname -a - name: Configure build - run: cmake -S $GITHUB_WORKSPACE -B $GITHUB_WORKSPACE/build - - name: Build - run: make -C $GITHUB_WORKSPACE/build - - name: Run unit tests - run: make -C $GITHUB_WORKSPACE/build test + run: cmake -S $GITHUB_WORKSPACE -B $GITHUB_WORKSPACE/build -DCMAKE_BUILD_TYPE=debug + - name: Test + run: make -C $GITHUB_WORKSPACE/build -j `nproc` e2e diff --git a/.github/workflows/fork.yaml b/.github/workflows/fork.yaml deleted file mode 100644 index f6dda018..00000000 --- a/.github/workflows/fork.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: Fork-specific CI - -on: - push: - branches: - - "**" - -permissions: - checks: write - pull-requests: write - -jobs: - full: - if: github.repository != 'facebook/bpfilter' - strategy: - fail-fast: false - matrix: - # This forces GitHub to print "fedora:41, x64" in the job name. - system: ["fedora:41"] - arch: ["X64"] - mode: ["release", "debug"] - container: "${{ matrix.system }}" - runs-on: ["ubuntu-latest"] - steps: - - name: Check out repository code - uses: actions/checkout@v4 - - name: Install dependencies (Fedora) - run: | - sudo dnf --disablerepo=* --enablerepo=fedora,updates --setopt=install_weak_deps=False -y install \ - bison bpftool clang clang-tools-extra cmake doxygen flex g++ git gcc jq lcov libasan libbpf-devel \ - libcmocka-devel libnl3-devel libubsan pkgconf python3-breathe python3-furo python3-linuxdoc \ - python3-sphinx - - name: Configure build - run: cmake -S $GITHUB_WORKSPACE -B $GITHUB_WORKSPACE/build -DCMAKE_BUILD_TYPE=${{ matrix.mode }} - - name: Build - run: make -C $GITHUB_WORKSPACE/build - # Skip unit tests, the GitHub-hosted runners' kernel is too old. - # Skip coverage as unit tests are not run. - - name: Check style - run: make -C $GITHUB_WORKSPACE/build check - - name: Generate documentation - run: make -C $GITHUB_WORKSPACE/build doc diff --git a/.github/workflows/pages.yaml b/.github/workflows/pages.yaml deleted file mode 100644 index 021fd1e7..00000000 --- a/.github/workflows/pages.yaml +++ /dev/null @@ -1,51 +0,0 @@ -name: Deploy static content to Pages - -on: - push: - branches: - - main - -permissions: - contents: read - pages: write - id-token: write - -# Allow only one concurrent deployment, skipping runs queued between the run -# in-progress and latest queued. However, do NOT cancel in-progress runs as -# we want to allow these production deployments to complete. -concurrency: - group: "pages" - cancel-in-progress: false - -jobs: - deploy: - if: github.repository == 'facebook/bpfilter' - container: "fedora:41" - runs-on: ubuntu-latest - steps: - - name: Setup Pages - uses: actions/configure-pages@v4 - - name: Check out repository code - uses: actions/checkout@v4 - - name: Install dependencies (Fedora) - run: | - sudo dnf --disablerepo=* --enablerepo=fedora,updates --setopt=install_weak_deps=False -y install \ - bison bpftool clang clang-tools-extra cmake doxygen flex g++ git gcc jq lcov libasan libbpf-devel \ - libcmocka-devel libnl3-devel libubsan pkgconf python3-breathe python3-furo python3-linuxdoc \ - python3-sphinx - - name: Configure build - run: cmake -S $GITHUB_WORKSPACE -B $GITHUB_WORKSPACE/build - - name: Generate documentation - run: make -C $GITHUB_WORKSPACE/build doc - - name: Fix permissions - run: | - chmod -c -R +rX "$GITHUB_WORKSPACE/build/doc/html" | while read line; do - echo "::warning title=Invalid file permissions automatically fixed::$line" - done - - name: Upload artifact - uses: actions/upload-pages-artifact@v3 - with: - path: build/doc/html - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v4 diff --git a/src/bpfilter/cgen/program.h b/src/bpfilter/cgen/program.h index b7552a4f..e9348180 100644 --- a/src/bpfilter/cgen/program.h +++ b/src/bpfilter/cgen/program.h @@ -135,24 +135,24 @@ struct bf_program_context { /** Argument passed to the BPF program, its content depends on the BPF * program type. */ - void *arg; + void bf_aligned(8) *arg; /** BPF dynamic pointer representing the packet data. Dynamic pointers are * used with every program type. */ - struct bpf_dynptr dynptr; + struct bpf_dynptr bf_aligned(8) dynptr; /** Total size of the packet. */ - uint64_t pkt_size; + uint64_t bf_aligned(8) pkt_size; /** Offset of the layer 3 protocol. */ - uint32_t l3_offset; + uint32_t bf_aligned(8) l3_offset; /** Offset of the layer 4 protocol. */ - uint32_t l4_offset; + uint32_t bf_aligned(8) l4_offset; /** On ingress, index of the input interface. On egress, index of the * output interface. */ - uint32_t ifindex; + uint32_t bf_aligned(8) ifindex; /** Layer 3 protocol, set when processing layer 2 protocol header. Required * to process the layer 3 header. */ diff --git a/src/bpfilter/cgen/stub.c b/src/bpfilter/cgen/stub.c index 9be6ce0e..b4366701 100644 --- a/src/bpfilter/cgen/stub.c +++ b/src/bpfilter/cgen/stub.c @@ -180,8 +180,12 @@ int bf_stub_parse_l3_hdr(struct bf_program *program) bf_swich_get(program, BF_ARG_4); EMIT_SWICH_OPTION(&swich, htobe16(ETH_P_IP), + BPF_STX_MEM(BPF_H, BF_REG_CTX, BF_ARG_4, + BF_PROG_CTX_OFF(l3_proto)), BPF_MOV64_IMM(BF_ARG_4, sizeof(struct iphdr))); EMIT_SWICH_OPTION(&swich, htobe16(ETH_P_IPV6), + BPF_STX_MEM(BPF_H, BF_REG_CTX, BF_ARG_4, + BF_PROG_CTX_OFF(l3_proto)), BPF_MOV64_IMM(BF_ARG_4, sizeof(struct ipv6hdr))); EMIT_SWICH_DEFAULT(&swich, BPF_MOV64_IMM(BF_ARG_4, 0), BPF_STX_MEM(BPF_H, BF_REG_CTX, BF_ARG_4, @@ -264,6 +268,7 @@ int bf_stub_parse_l3_hdr(struct bf_program *program) offsetof(struct ipv6hdr, nexthdr)), BPF_STX_MEM(BPF_B, BF_REG_CTX, BF_REG_1, BF_PROG_CTX_OFF(l4_proto))); + EMIT_SWICH_DEFAULT(&swich); r = bf_swich_generate(&swich); if (r) diff --git a/tests/benchmark/CMakeLists.txt b/tests/benchmark/CMakeLists.txt index 757e9780..1e20fe12 100644 --- a/tests/benchmark/CMakeLists.txt +++ b/tests/benchmark/CMakeLists.txt @@ -28,7 +28,6 @@ target_link_libraries(benchmark_bin add_custom_target(benchmark COMMAND - sudo $ --cli $ --daemon $ diff --git a/tests/benchmark/benchmark.cpp b/tests/benchmark/benchmark.cpp index 4b20e0e7..c1be2ecb 100644 --- a/tests/benchmark/benchmark.cpp +++ b/tests/benchmark/benchmark.cpp @@ -289,6 +289,7 @@ int setup(std::span args) err("failed to parse command line arguments: {}", errStr(r)); return r; } + /* const ::bf::Sources srcs(::bf::config.srcdir); @@ -307,9 +308,10 @@ int setup(std::span args) const auto pos = config.outfile.find(pattern); if (pos != ::std::string::npos) config.outfile.replace(pos, pattern.size(), config.gitrev); + */ - ::benchmark::AddCustomContext("gitrev", config.gitrev); - ::benchmark::AddCustomContext("gitdate", ::std::to_string(config.gitdate)); + ::benchmark::AddCustomContext("gitrev", ""); + ::benchmark::AddCustomContext("gitdate", ""); ::benchmark::AddCustomContext("bfcli", config.bfcli); ::benchmark::AddCustomContext("bpfilter", config.bpfilter); ::benchmark::AddCustomContext("srcdir", config.srcdir); diff --git a/tests/e2e/CMakeLists.txt b/tests/e2e/CMakeLists.txt index 25ad862e..33fdf040 100644 --- a/tests/e2e/CMakeLists.txt +++ b/tests/e2e/CMakeLists.txt @@ -40,8 +40,7 @@ set_property(TARGET e2e_bin PROPERTY POSITION_INDEPENDENT_CODE FALSE) add_custom_target(e2e COMMAND - sudo - $ + $ --bpfilter $ DEPENDS e2e_bin diff --git a/tests/harness/daemon.c b/tests/harness/daemon.c index a3687d0d..ea031aa1 100644 --- a/tests/harness/daemon.c +++ b/tests/harness/daemon.c @@ -26,7 +26,7 @@ int bf_test_daemon_init(struct bf_test_daemon *daemon, const char *path, uint32_t options) { - char *args[__builtin_ctz(_BF_TEST_DAEMON_LAST) + 1] = {}; + char *args[__builtin_ctz(_BF_TEST_DAEMON_LAST) + 1 + 6] = {}; size_t nargs = 0; bf_assert(daemon); @@ -40,6 +40,13 @@ int bf_test_daemon_init(struct bf_test_daemon *daemon, const char *path, if (options & BF_TEST_DAEMON_NO_NFTABLES) args[nargs++] = "--no-nftables"; + args[nargs++] = "--verbose"; + args[nargs++] = "debug"; + args[nargs++] = "--verbose"; + args[nargs++] = "bpf"; + args[nargs++] = "--verbose"; + args[nargs++] = "bytecode"; + return bf_test_process_init(&daemon->process, path, args, nargs); } diff --git a/tools/ci.yaml b/tools/ci.yaml new file mode 100644 index 00000000..cf4de3db --- /dev/null +++ b/tools/ci.yaml @@ -0,0 +1,84 @@ +name: Testing on CI + +on: + push: + branches: + - "**" + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +permissions: + contents: read + packages: write + +jobs: + create-container: + strategy: + matrix: + host: + - name: "ubuntu-latest" + arch: "x64" + - name: "4-core-ubuntu-arm" + arch: "arm64" + distribution: + - name: fedora + version: 41 + - name: fedora + version: 40 + - name: fedora + version: 39 + runs-on: ["${{ matrix.host.name }}"] + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Log into container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + - name: Build and push container image + id: build-and-push + uses: docker/build-push-action@v4 + with: + file: tools/${{ matrix.distribution.name }}-${{ matrix.distribution.version }}.Dockerfile + push: true + tags: ghcr.io/facebook/bpfilter:${{ matrix.distribution.name }}-${{ matrix.distribution.version }}-${{ matrix.host.arch}} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + build: + needs: create-container + runs-on: ["4-core-ubuntu-arm"] + container: ghcr.io/facebook/bpfilter:fedora-41-arm64 + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Configure build + run: cmake -S $GITHUB_WORKSPACE -B $GITHUB_WORKSPACE/build -DCMAKE_BUILD_TYPE=debug + - name: Build + run: make -C $GITHUB_WORKSPACE/build -j `nproc` bpfilter libbpfilter bfcli + + benchmark: + needs: create-container + runs-on: ["4-core-ubuntu-arm"] + container: + image: ghcr.io/facebook/bpfilter:fedora-41-arm64 + options: --privileged + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Configure build + run: cmake -S $GITHUB_WORKSPACE -B $GITHUB_WORKSPACE/build -DCMAKE_BUILD_TYPE=debug -DENABLE_BENCHMARK:BOOL=on + - name: Build + run: make -C $GITHUB_WORKSPACE/build -j `nproc` benchmark diff --git a/tools/fedora-39.Dockerfile b/tools/fedora-39.Dockerfile new file mode 100644 index 00000000..5c712e0f --- /dev/null +++ b/tools/fedora-39.Dockerfile @@ -0,0 +1,26 @@ +FROM fedora:39 + +RUN dnf --disablerepo=* --enablerepo=fedora,updates --setopt=install_weak_deps=False -y install \ + bison \ + bpftool \ + clang \ + clang-tools-extra \ + cmake \ + flex \ + libcmocka-devel \ + doxygen \ + git \ + jq \ + lcov \ + libasan \ + libbpf-devel \ + libnl3-devel \ + libubsan \ + python3-breathe \ + python3-furo \ + python3-linuxdoc \ + python3-scapy \ + python3-sphinx \ + pkgconf \ + google-benchmark-devel \ + libgit2-devel diff --git a/tools/fedora-40.Dockerfile b/tools/fedora-40.Dockerfile new file mode 100644 index 00000000..43b626ef --- /dev/null +++ b/tools/fedora-40.Dockerfile @@ -0,0 +1,26 @@ +FROM fedora:40 + +RUN dnf --disablerepo=* --enablerepo=fedora,updates --setopt=install_weak_deps=False -y install \ + bison \ + bpftool \ + clang \ + clang-tools-extra \ + cmake \ + flex \ + libcmocka-devel \ + doxygen \ + git \ + jq \ + lcov \ + libasan \ + libbpf-devel \ + libnl3-devel \ + libubsan \ + python3-breathe \ + python3-furo \ + python3-linuxdoc \ + python3-scapy \ + python3-sphinx \ + pkgconf \ + google-benchmark-devel \ + libgit2-devel diff --git a/tools/fedora-41.Dockerfile b/tools/fedora-41.Dockerfile new file mode 100644 index 00000000..c32d1899 --- /dev/null +++ b/tools/fedora-41.Dockerfile @@ -0,0 +1,26 @@ +FROM fedora:41 + +RUN dnf --disablerepo=* --enablerepo=fedora,updates --setopt=install_weak_deps=False -y install \ + bison \ + bpftool \ + clang \ + clang-tools-extra \ + cmake \ + flex \ + libcmocka-devel \ + doxygen \ + git \ + jq \ + lcov \ + libasan \ + libbpf-devel \ + libnl3-devel \ + libubsan \ + python3-breathe \ + python3-furo \ + python3-linuxdoc \ + python3-scapy \ + python3-sphinx \ + pkgconf \ + google-benchmark-devel \ + libgit2-devel diff --git a/tools/ubuntu-24.04.Dockerfile b/tools/ubuntu-24.04.Dockerfile new file mode 100644 index 00000000..2d608e64 --- /dev/null +++ b/tools/ubuntu-24.04.Dockerfile @@ -0,0 +1,32 @@ +FROM ubuntu:24.04 + +RUN apt-get update && \ + apt-get install -y \ + bison \ + linux-tools-common \ + clang \ + clang-tidy \ + clang-format \ + cmake \ + doxygen \ + flex \ + g++ \ + gcc \ + git \ + jq \ + lcov \ + libasan8 \ + libbpf-dev \ + libcmocka-dev \ + libnl-3-dev \ + libubsan1 \ + pkgconf \ + python3-breathe \ + python3-scapy \ + furo \ + python3-pip \ + python3-sphinx \ + libbenchmark-dev \ + libgit2-dev + +RUN pip install --break-system-packages linuxdoc diff --git a/tools/ubuntu-24.10.Dockerfile b/tools/ubuntu-24.10.Dockerfile new file mode 100644 index 00000000..af27f0fe --- /dev/null +++ b/tools/ubuntu-24.10.Dockerfile @@ -0,0 +1,32 @@ +FROM ubuntu:24.10 + +RUN apt-get update && \ + apt-get install -y \ + bison \ + linux-tools-common \ + clang \ + clang-tidy \ + clang-format \ + cmake \ + doxygen \ + flex \ + g++ \ + gcc \ + git \ + jq \ + lcov \ + libasan8 \ + libbpf-dev \ + libcmocka-dev \ + libnl-3-dev \ + libubsan1 \ + pkgconf \ + python3-breathe \ + python3-scapy \ + furo \ + python3-pip \ + python3-sphinx \ + libbenchmark-dev \ + libgit2-dev + +RUN pip install --break-system-packages linuxdoc