libcdb-cli: add --offline-only, refactor unstrip and add fetch pars…
#3222
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Continuous Integration | |
| on: [push, pull_request] | |
| jobs: | |
| test: | |
| strategy: | |
| matrix: | |
| python_version: ['3.10', '3.12'] | |
| os: [ubuntu-latest] | |
| include: | |
| - python_version: '2.7' | |
| os: ubuntu-22.04 | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 30 | |
| services: | |
| libcdb-cache: | |
| image: nginx | |
| volumes: | |
| - /home/runner/libcdb-cache:/var/cache/nginx | |
| ports: | |
| - 3000:3000 # https://debuginfod.elfutils.org proxy cache | |
| - 3001:3001 # https://libc.rip/ proxy cache | |
| - 3002:3002 # http://archive.ubuntu.com/ proxy cache | |
| - 3003:3003 # https://gitlab.com/ proxy cache | |
| env: | |
| DEBUGINFOD_URLS: http://localhost:3000/ | |
| PWN_LIBCRIP_URL: http://localhost:3001/ | |
| PWN_UBUNTU_ARCHIVE_URL: http://localhost:3002/ | |
| PWN_GITLAB_LIBCDB_URL: http://localhost:3003/ | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 20 | |
| - name: Git History | |
| run: | | |
| git fetch origin | |
| git log --oneline --graph -10 | |
| - name: Fix libcdb-cache permissions | |
| id: fix-perms | |
| run: | | |
| sudo chown -R runner:runner /home/runner/libcdb-cache | |
| echo "date=$(/bin/date -u "+%Y%m%d%H%M%S")" >> $GITHUB_OUTPUT | |
| - name: Cache for libcdb requests | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/libcdb-cache | |
| key: libcdb-python${{ matrix.python_version }}-${{ steps.fix-perms.outputs.date }} | |
| restore-keys: | | |
| libcdb-python${{ matrix.python_version }}- | |
| libcdb- | |
| - name: Install libcdb-cache service config | |
| run: | | |
| sudo chown -R 101:101 /home/runner/libcdb-cache | |
| container_id=$(docker ps --all --filter volume=/home/runner/libcdb-cache --no-trunc --format "{{.ID}}") | |
| docker cp ./travis/libcdb_nginx_cache.conf $container_id:/etc/nginx/nginx.conf | |
| docker restart $container_id | |
| - name: Install RPyC for gdb | |
| run: | | |
| # The version packaged in python3-rpyc is too old on Ubuntu 24.04 | |
| # We use ^6.0 from pip. | |
| sudo apt-get update && sudo apt-get install -y python3-pip gdb gdbserver | |
| /usr/bin/python -m pip install --break-system-packages rpyc || /usr/bin/python -m pip install rpyc | |
| gdb --batch --quiet --nx --nh --ex 'py import rpyc; print(rpyc.version.version)' | |
| - name: Cache for pip | |
| uses: actions/cache@v4 | |
| if: matrix.python_version == '2.7' | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ matrix.os }}-${{ matrix.python_version }}-cache-pip-${{ hashFiles('**/pyproject.toml', '**/requirements*.txt') }} | |
| restore-keys: ${{ matrix.os }}-${{ matrix.python_version }}-cache-pip- | |
| - name: Set up Python ${{ matrix.python_version }} | |
| if: matrix.python_version != '2.7' | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python_version }} | |
| cache: 'pip' | |
| cache-dependency-path: | | |
| **/pyproject.toml | |
| **/requirements*.txt | |
| - name: Set up Python 2.7 | |
| if: matrix.python_version == '2.7' | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y \ | |
| python2.7 python2.7-dev python2-pip-whl | |
| sudo ln -sf python2.7 /usr/bin/python | |
| export PYTHONPATH=`echo /usr/share/python-wheels/pip-*py2*.whl` | |
| sudo --preserve-env=PYTHONPATH python -m pip install --upgrade pip setuptools wheel | |
| sudo chown -R $USER /usr/local/lib/python2.7 | |
| - name: Verify tag against version | |
| if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') | |
| env: | |
| GITHUB_REF: ${{ github.event.ref }} | |
| run: | | |
| set -x | |
| GITHUB_TAG=${GITHUB_REF#refs/tags/} | |
| echo "$GITHUB_TAG" | grep -E '^[0-9.]*(beta[0-9])?$' | |
| vsetup=$(grep -o "version\\s*=\\s*[\"'].*[\"'],$" setup.py | grep -o "[0-9][^\"']*") | |
| vpwnlib=$(grep -o "__version__\\s*=\\s*[\"'].*[\"']$" pwnlib/version.py | grep -o "[0-9][^\"']*") | |
| [ "$vsetup" = "$vpwnlib" ] | |
| [ "$GITHUB_TAG" = "$vsetup" ] | |
| - name: Install Linux dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends -o Acquire::Retries=3 \ | |
| ash bash-static dash ksh mksh zsh \ | |
| gdb gdbserver socat \ | |
| binutils-multiarch qemu-user-static \ | |
| binutils-aarch64-linux-gnu \ | |
| binutils-arm-linux-gnueabihf \ | |
| binutils-mips-linux-gnu \ | |
| binutils-msp430 \ | |
| binutils-powerpc-linux-gnu \ | |
| binutils-s390x-linux-gnu \ | |
| binutils-sparc64-linux-gnu \ | |
| binutils-riscv64-linux-gnu \ | |
| gcc-multilib \ | |
| libc6-dbg \ | |
| elfutils \ | |
| patchelf | |
| - name: Testing Corefiles | |
| run: | | |
| ulimit -a | |
| ulimit -c unlimited | |
| cat /proc/sys/kernel/core_pattern | |
| cat /proc/sys/kernel/core_uses_pid | |
| ( cd $(mktemp -d); sh -c 'kill -11 $$' || true; ls -la ./*core* /var/crash/*.crash /var/lib/apport/coredump/core*) || true | |
| - name: Set up SSH | |
| run: | | |
| travis/ssh_setup.sh | |
| - name: Install dependencies | |
| run: | | |
| pip install --upgrade pip | |
| pip install --upgrade wheel build | |
| pip install --upgrade flake8 appdirs | |
| pip install --upgrade --editable . | |
| - name: Sanity checks | |
| run: PWNLIB_NOTERM=1 python -bb -c 'from pwn import *; print(pwnlib.term.term_mode)' | |
| - name: Install documentation dependencies | |
| run: pip install -r docs/requirements.txt | |
| - name: Manually install non-broken Unicorn | |
| if: matrix.python_version == '2.7' | |
| run: pip install unicorn==2.0.0rc7 | |
| - name: Disable yama ptrace_scope | |
| run: | | |
| echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope # required by some gdb doctests | |
| - name: Coverage doctests | |
| run: | | |
| # Python version installed using setup-python interferes with gdb's python | |
| # by setting LD_LIBRARY_PATH and gdb's python becoming unable to load built-in modules | |
| # like _socket. This is a workaround. | |
| unset LD_LIBRARY_PATH | |
| PWNLIB_NOTERM=1 python -bb -m coverage run -m sphinx -b doctest docs/source docs/build/doctest | |
| - name: Coverage running examples | |
| run: | | |
| export TERM=linux | |
| set -x | |
| python -bb travis/coverage_chdir.py examples/fmtstr examples/fmtstr/exploit.py | |
| python -bb travis/coverage_chdir.py examples/fmtstr examples/fmtstr/exploit2.py || : # can fail randomly? | |
| python -bb -m coverage run examples/asm.py | |
| python -bb -m coverage run examples/asm.py | |
| python -bb -m coverage run examples/text.py | |
| # for f in examples/sigreturn_corefile_*.py; do coverage run "$f"; done # XXX something is wrong | |
| - name: Coverage running commandline tools | |
| run: | | |
| export TERM=linux | |
| pwn() { ( set +x; cmd=$1; shift; PYTHONUNBUFFERED=1 exec python -bb -m coverage run -m pwnlib.commandline."$cmd" "$@" ) } | |
| set -x | |
| pwn cyclic 32 | |
| pwn cyclic -l 0x62616161 | |
| pwn cyclic -a ab | |
| echo | |
| pwn shellcraft --list |tail | |
| pwn shellcraft -l --syscalls |tail | |
| pwn shellcraft -l execve | |
| pwn shellcraft -l execve + exit | |
| pwn shellcraft --show i386.linux.loader_append | |
| pwn shellcraft --show i386.linux.loader_append + i386.linux.sh | |
| pwn shellcraft -f asm --color amd64.linux.sh | |
| pwn shellcraft -f asm --color amd64.linux.setreuid + amd64.linux.cat /etc/passwd | |
| pwn shellcraft -f asm --color amd64.linux.setreuid = amd64.linux.cat /key+secret --delim = | |
| pwn shellcraft -f elf amd64.linux.syscalls.exit 0 </dev/null |pwn hex | |
| pwn shellcraft -f elf amd64.linux.cat /etc/passwd + amd64.linux.syscalls.exit 0 </dev/null |pwn hex | |
| pwn shellcraft -f i --color amd64.linux.cat /etc/passwd </dev/null | |
| pwn shellcraft -f i --color amd64.linux.cat /etc/passwd + amd64.linux.sh </dev/null | |
| pwn shellcraft -f c amd64.linux.syscalls.exit 0 </dev/null | |
| pwn shellcraft -f c amd64.linux.cat /etc/passwd + amd64.linux.syscalls.exit 0 </dev/null | |
| pwn shellcraft -f str aarch64.linux.sh </dev/null | |
| pwn shellcraft -abr -f elf -o /dev/null amd64.linux.cat /etc/passwd </dev/null | |
| pwn shellcraft -nzr thumb.linux.syscalls.execve /bin/cat '["/bin/cat", "/etc/os-release"]' </dev/null | |
| pwn shellcraft -fp aarch64.trap | |
| pwn disasm --color ff3424c3ebfe | |
| pwn asm -f hex nop | |
| pwn hex ABCD | |
| pwn hex ABCD --separator ' ' | |
| pwn hex ABCD --prefix '\x' | |
| pwn hex ABCD -p '0x' -s ' ' | |
| pwn hex abcd | |
| pwn unhex 4141 4141 | |
| cat /dev/urandom | pwn phd --color -c 256 -s 2 | |
| pwn phd -l 0x3d --color=always /etc/os-release | |
| pwn checksec /bin/bash | |
| (ulimit -v 500000 && pwn checksec /bin/bash) | |
| pwn errno 2 | |
| pwn errno -1 | |
| pwn errno EADDRINUSE | |
| pwn constgrep -c freebsd -m ^PROT_ '3 + 4' | |
| pwn constgrep ^MAP_ 0 | |
| pwn constgrep -e O_RDWR | |
| pwn constgrep C | |
| pwn libcdb file /lib/x86_64-linux-gnu/libc.so.6 | |
| pwn libcdb lookup puts 5f0 __libc_start_main_ret d0a | |
| pwn libcdb hash b229d1da1e161f95e839cf90cded5f719e5de308 | |
| - name: Build source and wheel distributions | |
| if: matrix.python_version != '2.7' | |
| run: | | |
| python -m build | |
| - uses: actions/upload-artifact@v4 | |
| if: matrix.python_version == '3.10' | |
| with: | |
| name: packages | |
| path: dist/ | |
| include-hidden-files: true | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-${{ matrix.python_version }} | |
| path: .coverage* | |
| include-hidden-files: true | |
| - name: Fix libcdb-cache permissions | |
| run: | | |
| container_id=$(docker ps --filter volume=/home/runner/libcdb-cache --no-trunc --format "{{.ID}}") | |
| docker stop $container_id | |
| sudo chown -R runner:runner /home/runner/libcdb-cache | |
| windows-test: | |
| runs-on: windows-latest | |
| timeout-minutes: 30 | |
| continue-on-error: true | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Install dependencies | |
| run: | | |
| pip install --upgrade pip | |
| pip install --upgrade --editable . | |
| - name: Sanity checks | |
| run: | | |
| python -bb -c 'from pwn import *' | |
| python -bb examples/text.py | |
| upload-coverage: | |
| runs-on: ubuntu-latest | |
| needs: test | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 20 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| pattern: coverage-* | |
| merge-multiple: true | |
| - name: Install coveralls | |
| run: | | |
| pip install --break-system-packages tomli coveralls | |
| - name: Upload coverage to coveralls.io | |
| run: | | |
| coverage combine | |
| COVERALLS_REPO_TOKEN=PP20MEgztXIQJJTguQwe2jeCh6Bm4lkbv coveralls | |
| staging-merge: | |
| runs-on: ubuntu-latest | |
| if: github.repository_owner == 'Gallopsled' && github.event_name == 'push' && startsWith(github.event.ref, 'refs/heads/') && endsWith(github.event.ref, '-staging') | |
| needs: test | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 20 | |
| - name: Push changes to protected branch | |
| env: | |
| GITHUB_REF: ${{ github.event.ref }} | |
| run: | | |
| BRANCH=${GITHUB_REF#refs/heads/} | |
| TARGET=${BRANCH%-staging} | |
| git branch -f "$TARGET" | |
| git push origin "$TARGET" | |
| git push origin --delete "$BRANCH" | |
| pypi: | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') | |
| permissions: | |
| id-token: write | |
| needs: test | |
| steps: | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: packages | |
| path: dist | |
| - name: Publish package | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| - if: failure() | |
| run: ls -R |