Milestone 2.1

.gitattributes

@@ -0,0 +1,3 @@
+src/flamenco/types/fd_types.h linguist-generated=true
+src/flamenco/types/fd_types.c linguist-generated=true
+src/flamenco/runtime/tests/generated/*.h filter=lfs diff=lfs merge=lfs -text

.github/actions/deps/action.yml

@@ -0,0 +1,54 @@
+name: deps
+description: 'Build and cache dependencies'
+inputs:
+  deps-script-path:
+    description: 'Path of deps.sh script'
+    required: true
+    default: './deps.sh'
+  deps-bundle-path:
+    description: 'Path of deps-bundle.sh script'
+    required: true
+    default: './contrib/deps-bundle.sh'
+outputs: {}
+runs:
+  using: composite
+  steps:
+    - name: Has apt-get?
+      shell: bash
+      run: |
+        if command -v apt-get > /dev/null 2>&1; then
+          echo "HAS_APT_GET=1" >> $GITHUB_ENV
+        else
+          echo "HAS_APT_GET=0" >> $GITHUB_ENV
+        fi
+
+    - name: apt-get update
+      shell: bash
+      run: sudo apt-get update
+      if: env.HAS_APT_GET == '1'
+
+    - id: deps-sh-hash
+      shell: bash
+      run: sha256sum '${{ inputs.deps-script-path }}' | awk '{print "HASH=" $1}' >> "$GITHUB_OUTPUT"
+
+    - id: deps-sh-cache
+      uses: actions/cache@v3
+      with:
+        path: deps-bundle.tar.zst
+        key: ${{ runner.os }}-deps-sh-${{ steps.deps-sh-hash.outputs.HASH }}
+
+    - name: Install system level dependencies
+      shell: bash
+      run: FD_AUTO_INSTALL_PACKAGES=1 '${{ inputs.deps-script-path }}' check
+
+    - name: Install dependencies from cache
+      shell: bash
+      run: tar -Izstd -xvf deps-bundle.tar.zst
+      if: steps.deps-sh-cache.outputs.cache-hit == 'true'
+
+    - name: Install dependencies from scratch
+      shell: bash
+      run: |
+        FD_AUTO_INSTALL_PACKAGES=1 '${{ inputs.deps-script-path }}' install
+        '${{ inputs.deps-bundle-path }}'
+      if: steps.deps-sh-cache.outputs.cache-hit != 'true'

.github/actions/hugepages/action.yml

@@ -0,0 +1,18 @@
+name: hugepages
+description: 'Setup 1 GiB gigantic pages'
+inputs:
+  count:
+    description: 'Number of huge pages'
+    required: true
+    default: '64'
+outputs: {}
+runs:
+  using: composite
+  steps:
+    - shell: bash
+      run: |
+        set -x
+        sudo src/util/shmem/fd_shmem_cfg fini || true
+        sudo src/util/shmem/fd_shmem_cfg init 0666 $USER "" || true
+        sudo src/util/shmem/fd_shmem_cfg alloc '${{ inputs.count }}' gigantic 0
+        sudo chown -R $USER:$USER /mnt/.fd

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows-disabled/coverage.yml

@@ -0,0 +1,119 @@
+name: All Coverage
+on:
+  schedule:
+    - cron: '0 0 * * *'
+  push:
+    branches:
+        - main
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+jobs:
+  all-coverage:
+    name: All Coverage
+    runs-on:
+      group: github-v1
+    environment:
+      name: github-pages
+      url: ${{ steps.pages-deploy.outputs.page_url }}
+    env:
+      CC: clang
+      EXTRAS: llvm-cov
+    steps:
+      - name: Install Dependencies
+        run: sudo apt-get update && sudo apt-get install -y llvm lcov
+
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - uses: ./.github/actions/deps
+      - uses: ./.github/actions/hugepages
+
+      - name: Build low unit tests
+        env:
+          MACHINE: linux_clang_combi_low
+        run: make -j -Otarget unit-test
+
+      - name: Run low unit tests
+        run: |
+          sudo prlimit --pid $$ --memlock=-1:-1
+          ./test.sh -j --page-sz gigantic
+
+      - name: Build high unit tests
+        env:
+          MACHINE: linux_clang_combi_low
+        run: make -j -Otarget unit-test
+
+      - name: Run high unit tests
+        run: |
+          sudo prlimit --pid $$ --memlock=-1:-1
+          ./test.sh -j --page-sz gigantic
+
+      - name: Make Test Coverage Report
+        run: |
+          make combicov-report
+          mkdir -p build/pages/
+          mv build/combi-cov/html build/pages/cov
+
+      - name: 'Authenticate to Google Cloud'
+        uses: 'google-github-actions/auth@v1'
+        with:
+          credentials_json: ${{ secrets.FUZZ_SERVICE_ACCT_JSON_BUNDLE }}
+
+      # Not using fuzzbot-builder because llvm-cov must be ran in the same environment otherwise paths will not match
+      - name: Build Fuzz Tests
+        env:
+          # Todo: use a matrix strategy
+          MACHINE: linux_clang_combi_high
+          EXTRAS: fuzz asan
+        run: make -j -Otarget fuzz-test
+
+      - name: Fetch Corpus, Generate Fuzzing Coverage
+        run: ./.github/workflows/scripts/fuzzcov_generate.sh build/linux/clang/combi/high/fuzz-test/
+
+      - name: Publish Fuzzing Coverage to codecov.io
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+        run: ./.github/workflows/scripts/fuzzcov_publish_codecov.sh
+
+      - name: Generate HTML Fuzzing Coverage Reports
+        run: ./.github/workflows/scripts/fuzzcov_genhtml.sh "${{ github.sha }}"
+
+      - name: Drop an Index
+        run: |
+          cat <<EOS > build/pages/index.html
+          <!DOCTYPE html>
+          <html>
+          <head>
+          <title>🔥💃 Pages</title>
+          </head>
+
+          <body>
+          <h1>🔥💃 Pages</h1>
+          <h2>Links 🔗</h2>
+          <ul>
+          <li><a href="./cov/">Test Coverage</li>
+          <li><a href="./fuzzcov/">Fuzzing Coverage</li>
+          </ul>
+          </body>
+
+          </html>
+          EOS
+
+      - name: Setup Pages
+        uses: actions/configure-pages@v3
+
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v2
+        with:
+          # Upload entire repository
+          path: './build/pages/'
+
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v2

.github/workflows-disabled/test_fuzz_regressions.yml

@@ -0,0 +1,28 @@
+name: Test Fuzz Regressions
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  make-fuzz:
+    name: Test Fuzz Regressions on ${{ matrix.feature_set }}
+    strategy:
+      matrix:
+        feature_set: [modern]
+    runs-on:
+      group: github-v1
+    env:
+      MACHINE: linux_clang_combi_${{ matrix.feature_set }}
+      EXTRAS: fuzz asan ubsan
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: ./.github/actions/deps
+
+      - name: Build Fuzz Tests
+        run: make -j -Otarget fuzz-test
+
+      - name: Run Fuzz Tests Against Corpora
+        run: make -k -j -Otarget run-fuzz-test

.github/workflows/_publish_to_clusterfuzz.yml

@@ -0,0 +1,41 @@
+name: Clusterfuzz
+on:
+  workflow_call:
+  workflow_dispatch:
+jobs:
+  publish:
+    name: Build Fuzzers for ${{ matrix.feature_set }} feature set
+    strategy:
+      matrix:
+        feature_set: [modern, highend]
+    runs-on:
+      group: github-v1
+    env:
+      MACHINE: linux_clang_combi_${{ matrix.feature_set }}
+      EXTRAS: fuzz asan ubsan
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: ./.github/actions/hugepages
+      - uses: ./.github/actions/deps
+
+      - run: sudo apt update && sudo apt install -y zip
+
+      - uses: firedancer-io/fuzzbot-builder@main
+        name: Build fuzz tests
+        with:
+          command: make -j -Otarget fuzz-test
+
+      - name: List Artifacts
+        run: |
+          ls build/linux/clang/combi/${{ matrix.feature_set }}/fuzz-test
+
+      - uses: firedancer-io/clusterfuzz-action@main
+        name: Upload fuzz targets to ClusterFuzz
+        with:
+          bucket-name: firedancer-builds.isol-clusterfuzz.appspot.com
+          artifact-dir: build/linux/clang/combi/${{ matrix.feature_set }}/fuzz-test
+          object-prefix: main/libfuzzer/${{ matrix.feature_set }}/firedancer
+          project-id: isol-clusterfuzz
+          qualifier: ${{ matrix.feature_set }}
+          service-account-credentials: ${{ secrets.FUZZ_SERVICE_ACCT_JSON_BUNDLE }}

.github/workflows/_test_fuzz_regressions.yml

@@ -0,0 +1,28 @@
+name: Test Fuzz Regressions
+on:
+  workflow_call:
+  workflow_dispatch:
+jobs:
+  test-fuzz-regressions:
+    name: Test Fuzz Regressions on ${{ matrix.feature_set }}
+    strategy:
+      matrix:
+        feature_set: [modern, highend]
+    runs-on:
+      group: ${{ matrix.feature_set == 'highend' && 'rhel85-icelake' || 'github-v1' }}
+    env:
+      MACHINE: linux_clang_combi_${{ matrix.feature_set }}
+      EXTRAS: fuzz asan ubsan
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: ./.github/actions/deps
+      - uses: ./.github/actions/hugepages
+
+      - name: Build Fuzz Tests
+        run: make -j -Otarget fuzz-test
+
+      - name: Run Fuzz Tests Against Corpora
+        run: |
+          sudo prlimit --pid $$ --memlock=-1:-1
+          make -k -j -Otarget run-fuzz-test

.github/workflows/_test_unit.yml

@@ -0,0 +1,33 @@
+name: Test Unit
+on:
+  workflow_call:
+  workflow_dispatch:
+jobs:
+  test-unit:
+    continue-on-error: true
+    strategy:
+      matrix:
+        feature_set: [modern, highend]
+        compiler: [gcc, clang]
+      fail-fast: true
+    runs-on:
+      group: ${{ matrix.feature_set == 'highend' && 'rhel85-icelake' || 'github-v1' }}
+    env:
+      MACHINE: linux_clang_combi_${{ matrix.feature_set }}
+      CC: ${{ matrix.compiler }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - uses: ./.github/actions/deps
+
+      - name: Build unit tests
+        run: make -j unit-test
+
+      - uses: ./.github/actions/hugepages
+
+      - name: Run unit tests
+        run: |
+          sudo prlimit --pid $$ --memlock=-1:-1
+          ./test.sh -j --page-sz gigantic

.github/workflows/_test_unit_with_sanitizers.yml

@@ -0,0 +1,31 @@
+name: Test Unit w/ Sanitizers
+on:
+  workflow_call:
+  workflow_dispatch:
+jobs:
+  test-with-sanitizers:
+    # Only run sanitizer tests if the gcc tests passed
+    continue-on-error: true
+    strategy:
+      matrix:
+        feature_set: [modern, highend]
+    runs-on:
+      group: ${{ matrix.feature_set == 'highend' && 'rhel85-icelake' || 'github-v1' }}
+    env:
+      MACHINE: linux_clang_combi_${{ matrix.feature_set }}
+      EXTRAS: asan ubsan
+    steps:
+      - run: lscpu
+
+      - uses: actions/checkout@v4
+
+      - uses: ./.github/actions/deps
+      - uses: ./.github/actions/hugepages
+
+      - name: Build unit tests
+        run: make -j -Otarget unit-test
+
+      - name: Run unit tests
+        run: |
+          sudo prlimit --pid $$ --memlock=-1:-1
+          ./test.sh -j --page-sz gigantic