diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 8f8e0fb65..258c51299 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -39,7 +39,7 @@ You should see `[DEBUG]` statements that show what's happening behind the scenes ## Verify on Ubuntu -If possible, please verify that your issue occurs on 64-bit Ubuntu 18.04. We provide a Dockerfile based on Ubuntu 18.04 via `docker.io` to make this super simple, no VM required! +If possible, please verify that your issue occurs on 64-bit Ubuntu 22.04. We provide a Dockerfile based on Ubuntu 22.04 via `docker.io` to make this super simple, no VM required! ```sh # Download the Docker image diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index d8e64c172..e066f94e2 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -6,24 +6,21 @@ jobs: android-test: strategy: matrix: - python-version: [3.8] + python-version: ['3.10'] os: [ubuntu-latest] runs-on: ${{ matrix.os }} timeout-minutes: 30 steps: - uses: actions/checkout@v4 - - name: Cache for pip - uses: actions/cache@v4 - id: cache-pip - with: - path: ~/.cache/pip - key: ${{ matrix.os }}-cache-pip - - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: | + **/pyproject.toml + **/requirements*.txt - name: Install Linux dependencies run: | @@ -35,6 +32,19 @@ jobs: binutils-arm-linux-gnueabihf \ libc6-dbg + - name: Cache for avd + uses: actions/cache@v4 + id: cache-avd + with: + path: | + ~/.android + /usr/local/lib/android/sdk/emulator + /usr/local/lib/android/sdk/platform-tools + /usr/local/lib/android/sdk/system-images + key: ${{ matrix.os }}-cache-avd-${{ hashFiles('travis/setup_avd*.sh') }} + restore-keys: | + ${{ matrix.os }}-cache-avd- + - name: Install Android AVD run: | sudo usermod -aG kvm $USER diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a1bd9b651..23d0c2309 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,10 +5,28 @@ jobs: test: strategy: matrix: - python_version: ['2.7', '3.10'] + python_version: ['3.10', '3.12'] os: [ubuntu-latest] + include: + - python_version: '2.7' + os: ubuntu-22.04 runs-on: ${{ matrix.os }} timeout-minutes: 30 + services: + libcdb-cache: + image: nginx + volumes: + - /home/runner/libcdb-cache:/var/cache/nginx + ports: + - 3000:3000 # https://debuginfod.elfutils.org proxy cache + - 3001:3001 # https://libc.rip/ proxy cache + - 3002:3002 # http://archive.ubuntu.com/ proxy cache + - 3003:3003 # https://gitlab.com/ proxy cache + env: + DEBUGINFOD_URLS: http://localhost:3000/ + PWN_LIBCRIP_URL: http://localhost:3001/ + PWN_UBUNTU_ARCHIVE_URL: http://localhost:3002/ + PWN_GITLAB_LIBCDB_URL: http://localhost:3003/ steps: - uses: actions/checkout@v4 with: @@ -18,26 +36,54 @@ jobs: run: | git fetch origin git log --oneline --graph -10 + + - name: Fix libcdb-cache permissions + id: fix-perms + run: | + sudo chown -R runner:runner /home/runner/libcdb-cache + echo "date=$(/bin/date -u "+%Y%m%d%H%M%S")" >> $GITHUB_OUTPUT + + - name: Cache for libcdb requests + uses: actions/cache@v4 + with: + path: ~/libcdb-cache + key: libcdb-python${{ matrix.python_version }}-${{ steps.fix-perms.outputs.date }} + restore-keys: | + libcdb-python${{ matrix.python_version }}- + libcdb- + + - name: Install libcdb-cache service config + run: | + sudo chown -R 101:101 /home/runner/libcdb-cache + container_id=$(docker ps --all --filter volume=/home/runner/libcdb-cache --no-trunc --format "{{.ID}}") + docker cp ./travis/libcdb_nginx_cache.conf $container_id:/etc/nginx/nginx.conf + docker restart $container_id - name: Install RPyC for gdb run: | - # The version packaged in python3-rpyc is too old on Ubuntu 22.04 + # The version packaged in python3-rpyc is too old on Ubuntu 24.04 + # We use ^6.0 from pip. sudo apt-get update && sudo apt-get install -y python3-pip gdb gdbserver - /usr/bin/python -m pip install rpyc + /usr/bin/python -m pip install --break-system-packages rpyc || /usr/bin/python -m pip install rpyc gdb --batch --quiet --nx --nh --ex 'py import rpyc; print(rpyc.version.version)' - + - name: Cache for pip uses: actions/cache@v4 - id: cache-pip + if: matrix.python_version == '2.7' with: path: ~/.cache/pip - key: ${{ matrix.os }}-cache-pip + key: ${{ matrix.os }}-${{ matrix.python_version }}-cache-pip-${{ hashFiles('**/pyproject.toml', '**/requirements*.txt') }} + restore-keys: ${{ matrix.os }}-${{ matrix.python_version }}-cache-pip- - name: Set up Python ${{ matrix.python_version }} if: matrix.python_version != '2.7' uses: actions/setup-python@v5 with: python-version: ${{ matrix.python_version }} + cache: 'pip' + cache-dependency-path: | + **/pyproject.toml + **/requirements*.txt - name: Set up Python 2.7 if: matrix.python_version == '2.7' @@ -203,7 +249,7 @@ jobs: python -m build - uses: actions/upload-artifact@v4 - if: matrix.python_version != '2.7' + if: matrix.python_version == '3.10' with: name: packages path: dist/ @@ -214,6 +260,12 @@ jobs: name: coverage-${{ matrix.python_version }} path: .coverage* include-hidden-files: true + + - name: Fix libcdb-cache permissions + run: | + container_id=$(docker ps --filter volume=/home/runner/libcdb-cache --no-trunc --format "{{.ID}}") + docker stop $container_id + sudo chown -R runner:runner /home/runner/libcdb-cache windows-test: runs-on: windows-latest @@ -252,7 +304,7 @@ jobs: - name: Install coveralls run: | - pip install tomli coveralls + pip install --break-system-packages tomli coveralls - name: Upload coverage to coveralls.io run: | diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d41f05912..2fa2d9f30 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -5,23 +5,21 @@ jobs: lint: strategy: matrix: - python-version: [3.8] + python-version: ['3.10'] os: [ubuntu-latest] runs-on: ${{ matrix.os }} timeout-minutes: 30 steps: - uses: actions/checkout@v4 - - name: Cache for pip - uses: actions/cache@v4 - id: cache-pip - with: - path: ~/.cache/pip - key: ${{ matrix.os }}-cache-pip - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: | + **/pyproject.toml + **/requirements*.txt - name: Critical lint run: | diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index c65003023..ee21b8614 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -5,23 +5,21 @@ jobs: build: strategy: matrix: - python-version: [3.8] + python-version: ['3.10'] os: [ubuntu-latest] runs-on: ${{ matrix.os }} timeout-minutes: 30 steps: - uses: actions/checkout@v4 - - name: Cache for pip - uses: actions/cache@v4 - id: cache-pip - with: - path: ~/.cache/pip - key: ${{ matrix.os }}-cache-pip - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: | + **/pyproject.toml + **/requirements*.txt - name: PyLint run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bf92d521..4cbb7dca0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -79,6 +79,9 @@ The table below shows which release corresponds to each branch, and what date th - [#2444][2444] Add `ELF.close()` to release resources - [#2413][2413] libcdb: improve the search speed of `search_by_symbol_offsets` in local libc-database - [#2470][2470] Fix waiting for gdb under WSL2 +- [#2479][2479] Support extracting libraries from Docker image in `pwn template` +- [#2483][2483] Only print `checksec` output of `ELF.libc` when it was printed for the `ELF` already +- [#2482][2482] Throw error when using `sni` and setting `server_hostname` manually in `remote` - [#2478][2478] libcdb-cli: add `--offline-only`, refactor unstrip and add fetch parser for download libc-database [2471]: https://github.com/Gallopsled/pwntools/pull/2471 @@ -87,6 +90,9 @@ The table below shows which release corresponds to each branch, and what date th [2444]: https://github.com/Gallopsled/pwntools/pull/2444 [2413]: https://github.com/Gallopsled/pwntools/pull/2413 [2470]: https://github.com/Gallopsled/pwntools/pull/2470 +[2479]: https://github.com/Gallopsled/pwntools/pull/2479 +[2483]: https://github.com/Gallopsled/pwntools/pull/2483 +[2482]: https://github.com/Gallopsled/pwntools/pull/2482 [2478]: https://github.com/Gallopsled/pwntools/pull/2478 ## 4.14.0 (`beta`) diff --git a/README.md b/README.md index 7bd8fdf99..464b007cb 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ To get you started, we've provided some example solutions for past CTF challenge # Installation -Pwntools is best supported on 64-bit Ubuntu LTS releases (14.04, 16.04, 18.04, and 20.04). Most functionality should work on any Posix-like distribution (Debian, Arch, FreeBSD, OSX, etc.). +Pwntools is best supported on 64-bit Ubuntu LTS releases (18.04, 20.04, 22.04, and 24.04). Most functionality should work on any Posix-like distribution (Debian, Arch, FreeBSD, OSX, etc.). Python3 is suggested, but Pwntools still works with Python 2.7. Most of the functionality of pwntools is self-contained and Python-only. You should be able to get running quickly with diff --git a/pwnlib/adb/adb.py b/pwnlib/adb/adb.py index 84bb89213..45d27875c 100644 --- a/pwnlib/adb/adb.py +++ b/pwnlib/adb/adb.py @@ -123,7 +123,7 @@ def current_device(any=False): >>> device = adb.current_device(any=True) >>> device # doctest: +ELLIPSIS - AdbDevice(serial='emulator-5554', type='device', port='emulator', product='sdk_...phone_...', model='...', device='generic...') + AdbDevice(serial='emulator-5554', type='device', port='emulator', product='sdk_...phone..._...', model='...', device='...') >>> device.port 'emulator' """ @@ -259,7 +259,7 @@ class AdbDevice(Device): >>> device.os 'android' >>> device.product # doctest: +ELLIPSIS - 'sdk_...phone_...' + 'sdk_...phone..._...' >>> device.serial 'emulator-5554' """ @@ -880,7 +880,7 @@ def which(name, all = False, *a, **kw): >>> adb.which('sh') '/system/bin/sh' >>> adb.which('sh', all=True) - ['/system/bin/sh'] + ['/system/bin/sh', '/vendor/bin/sh'] >>> adb.which('foobar') is None True @@ -988,7 +988,7 @@ def proc_exe(pid): :skipif: skip_android >>> adb.proc_exe(1) - b'/init' + b'/system/bin/init' """ with context.quiet: io = process(['realpath','/proc/%d/exe' % pid]) @@ -1365,7 +1365,7 @@ def compile(source): >>> filename = adb.compile(temp) >>> sent = adb.push(filename, "/data/local/tmp") >>> adb.process(sent).recvall() # doctest: +ELLIPSIS - b'... /system/lib64/libc.so\n...' + b'... /system/lib64/libc++.so\n...' """ ndk_build = misc.which('ndk-build') diff --git a/pwnlib/asm.py b/pwnlib/asm.py index b95288acd..1bbaccbef 100644 --- a/pwnlib/asm.py +++ b/pwnlib/asm.py @@ -908,8 +908,8 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True): 0: b8 17 00 00 00 mov eax, 0x17 >>> print(disasm(unhex('48c7c017000000'), arch = 'amd64')) 0: 48 c7 c0 17 00 00 00 mov rax, 0x17 - >>> print(disasm(unhex('04001fe552009000'), arch = 'arm')) - 0: e51f0004 ldr r0, [pc, #-4] ; 0x4 + >>> print(disasm(unhex('04001fe552009000'), arch = 'arm')) # doctest: +ELLIPSIS + 0: e51f0004 ldr r0, [pc, #-4] ... 4: 00900052 addseq r0, r0, r2, asr r0 >>> print(disasm(unhex('4ff00500'), arch = 'thumb', bits=32)) 0: f04f 0005 mov.w r0, #5 diff --git a/pwnlib/commandline/asm.py b/pwnlib/commandline/asm.py index 03c51a6a2..b130228ae 100644 --- a/pwnlib/commandline/asm.py +++ b/pwnlib/commandline/asm.py @@ -137,4 +137,4 @@ def main(args): args.output.write(b'\n') if __name__ == '__main__': - pwnlib.commandline.common.main(__file__) + pwnlib.commandline.common.main(__file__, main) diff --git a/pwnlib/commandline/checksec.py b/pwnlib/commandline/checksec.py index 9c97d6a05..18da2511f 100644 --- a/pwnlib/commandline/checksec.py +++ b/pwnlib/commandline/checksec.py @@ -41,4 +41,4 @@ def main(args): print("{name}: {error}".format(name=f.name, error=e)) if __name__ == '__main__': - common.main(__file__) + common.main(__file__, main) diff --git a/pwnlib/commandline/common.py b/pwnlib/commandline/common.py index 75edfdcb8..3ce0a0fad 100644 --- a/pwnlib/commandline/common.py +++ b/pwnlib/commandline/common.py @@ -25,8 +25,18 @@ def context_arg(arg): prog='pwn') parser_commands = parser.add_subparsers(dest='command') -def main(file=sys.argv[0]): - import pwnlib.commandline.main +def main(file=sys.argv[0], command_main=None): name = os.path.splitext(os.path.basename(file))[0] + if command_main is None: + import importlib + command_main = importlib.import_module('pwnlib.commandline.%s' % name).main sys.argv.insert(1, name) - pwnlib.commandline.main.main() + entrypoint({name: command_main}) + +def entrypoint(commands): + if len(sys.argv) < 2: + parser.print_usage() + sys.exit() + args = parser.parse_args() + with context.local(log_console = sys.stderr): + commands[args.command](args) diff --git a/pwnlib/commandline/constgrep.py b/pwnlib/commandline/constgrep.py index d9341f5c4..ace70f4ed 100644 --- a/pwnlib/commandline/constgrep.py +++ b/pwnlib/commandline/constgrep.py @@ -133,4 +133,4 @@ def main(args): print('(%s) == %s' % (' | '.join(k for v, k in good), args.constant)) if __name__ == '__main__': - pwnlib.commandline.common.main(__file__) + pwnlib.commandline.common.main(__file__, main) diff --git a/pwnlib/commandline/cyclic.py b/pwnlib/commandline/cyclic.py index ff012a359..c7a5060f6 100644 --- a/pwnlib/commandline/cyclic.py +++ b/pwnlib/commandline/cyclic.py @@ -107,4 +107,4 @@ def main(args): out.write(b'\n') if __name__ == '__main__': - pwnlib.commandline.common.main(__file__) + pwnlib.commandline.common.main(__file__, main) diff --git a/pwnlib/commandline/debug.py b/pwnlib/commandline/debug.py index fe5fca6f5..15593f915 100644 --- a/pwnlib/commandline/debug.py +++ b/pwnlib/commandline/debug.py @@ -102,4 +102,4 @@ def main(args): gdb.debug(target, gdbscript=gdbscript, sysroot=args.sysroot).interactive() if __name__ == '__main__': - pwnlib.commandline.common.main(__file__) + pwnlib.commandline.common.main(__file__, main) diff --git a/pwnlib/commandline/disablenx.py b/pwnlib/commandline/disablenx.py index 29839c0f8..14a7d6531 100644 --- a/pwnlib/commandline/disablenx.py +++ b/pwnlib/commandline/disablenx.py @@ -24,4 +24,4 @@ def main(args): ELF(e.path) if __name__ == '__main__': - pwnlib.commandline.common.main(__file__) + pwnlib.commandline.common.main(__file__, main) diff --git a/pwnlib/commandline/disasm.py b/pwnlib/commandline/disasm.py index b304393d2..3cf60b148 100644 --- a/pwnlib/commandline/disasm.py +++ b/pwnlib/commandline/disasm.py @@ -110,4 +110,4 @@ def main(args): print(disasm(dat, vma=safeeval.const(args.address))) if __name__ == '__main__': - pwnlib.commandline.common.main(__file__) + pwnlib.commandline.common.main(__file__, main) diff --git a/pwnlib/commandline/elfdiff.py b/pwnlib/commandline/elfdiff.py index 48afef09f..2ea307ad9 100644 --- a/pwnlib/commandline/elfdiff.py +++ b/pwnlib/commandline/elfdiff.py @@ -59,4 +59,4 @@ def main(a): print(diff(x, y)) if __name__ == '__main__': - pwnlib.commandline.common.main(__file__) + pwnlib.commandline.common.main(__file__, main) diff --git a/pwnlib/commandline/elfpatch.py b/pwnlib/commandline/elfpatch.py index 10a5adc24..ee8584a9e 100644 --- a/pwnlib/commandline/elfpatch.py +++ b/pwnlib/commandline/elfpatch.py @@ -34,4 +34,4 @@ def main(a): getattr(sys.stdout, 'buffer', sys.stdout).write(elf.get_data()) if __name__ == '__main__': - pwnlib.commandline.common.main(__file__) + pwnlib.commandline.common.main(__file__, main) diff --git a/pwnlib/commandline/errno.py b/pwnlib/commandline/errno.py index d3a05e37a..2090fbbe2 100644 --- a/pwnlib/commandline/errno.py +++ b/pwnlib/commandline/errno.py @@ -46,4 +46,4 @@ def main(args): print(os.strerror(value)) if __name__ == '__main__': - common.main(__file__) + common.main(__file__, main) diff --git a/pwnlib/commandline/hex.py b/pwnlib/commandline/hex.py index d538af246..d36291054 100644 --- a/pwnlib/commandline/hex.py +++ b/pwnlib/commandline/hex.py @@ -50,4 +50,4 @@ def main(args): print(encoded) if __name__ == '__main__': - common.main(__file__) + common.main(__file__, main) diff --git a/pwnlib/commandline/libcdb.py b/pwnlib/commandline/libcdb.py index 2ab61e8be..70ccaa1a6 100644 --- a/pwnlib/commandline/libcdb.py +++ b/pwnlib/commandline/libcdb.py @@ -288,4 +288,4 @@ def main(args): if __name__ == '__main__': - pwnlib.commandline.common.main(__file__) + pwnlib.commandline.common.main(__file__, main) diff --git a/pwnlib/commandline/main.py b/pwnlib/commandline/main.py index 5cfbdd5b7..9382c43c7 100644 --- a/pwnlib/commandline/main.py +++ b/pwnlib/commandline/main.py @@ -1,7 +1,5 @@ from __future__ import absolute_import -import sys - from pwnlib.commandline import asm from pwnlib.commandline import checksec from pwnlib.commandline import common @@ -23,8 +21,7 @@ from pwnlib.commandline import unhex from pwnlib.commandline import update from pwnlib.commandline import version -from pwnlib.commandline.common import parser -from pwnlib.context import context +from pwnlib.commandline.common import parser as parser commands = { 'asm': asm.main, @@ -50,12 +47,7 @@ } def main(): - if len(sys.argv) < 2: - parser.print_usage() - sys.exit() - args = parser.parse_args() - with context.local(log_console = sys.stderr): - commands[args.command](args) + common.entrypoint(commands) if __name__ == '__main__': main() diff --git a/pwnlib/commandline/phd.py b/pwnlib/commandline/phd.py index 7f3891e0f..1be34d969 100644 --- a/pwnlib/commandline/phd.py +++ b/pwnlib/commandline/phd.py @@ -109,4 +109,4 @@ def main(args): pass if __name__ == '__main__': - pwnlib.commandline.common.main(__file__) + pwnlib.commandline.common.main(__file__, main) diff --git a/pwnlib/commandline/pwnstrip.py b/pwnlib/commandline/pwnstrip.py index fcc8df171..b7601b605 100644 --- a/pwnlib/commandline/pwnstrip.py +++ b/pwnlib/commandline/pwnstrip.py @@ -53,4 +53,4 @@ def main(args): args.output.write(result) if __name__ == '__main__': - pwnlib.commandline.common.main(__file__) + pwnlib.commandline.common.main(__file__, main) diff --git a/pwnlib/commandline/scramble.py b/pwnlib/commandline/scramble.py index 8b5043be4..4d4b31cda 100644 --- a/pwnlib/commandline/scramble.py +++ b/pwnlib/commandline/scramble.py @@ -110,4 +110,4 @@ def main(args): if __name__ == '__main__': - pwnlib.commandline.common.main(__file__) + pwnlib.commandline.common.main(__file__, main) diff --git a/pwnlib/commandline/shellcraft.py b/pwnlib/commandline/shellcraft.py index 948275634..ae24e4c1f 100644 --- a/pwnlib/commandline/shellcraft.py +++ b/pwnlib/commandline/shellcraft.py @@ -384,4 +384,4 @@ def main(args): args.out.write(code) if __name__ == '__main__': - pwnlib.commandline.common.main(__file__) + pwnlib.commandline.common.main(__file__, main) diff --git a/pwnlib/commandline/template.py b/pwnlib/commandline/template.py index 5cd6c7341..faea7ca1d 100644 --- a/pwnlib/commandline/template.py +++ b/pwnlib/commandline/template.py @@ -1,9 +1,12 @@ from __future__ import absolute_import from __future__ import division +from __future__ import print_function from pwn import * from pwnlib.commandline import common +from pwnlib.util.misc import which, parse_ldd_output, write +from sys import stderr from mako.lookup import TemplateLookup, Template parser = common.parser_commands.add_parser( @@ -32,18 +35,100 @@ os.path.join(printable_data_path, "templates", "pwnup.mako")) parser.add_argument('--no-auto', help='Do not automatically detect missing binaries', action='store_false', dest='auto') +def get_docker_image_libraries(): + """Tries to retrieve challenge libraries from a Docker image built from the Dockerfile in the current working directory. + + The libraries are retrieved by parsing the output of running ldd on /bin/sh. + Supports regular Docker images as well as jail images. + """ + with log.progress("Extracting challenge libraries from Docker image") as progress: + if not which("docker"): + progress.failure("docker command not found") + return None, None + # maps jail image name to the root directory of the child image + jail_image_to_chroot_dir = { + "pwn.red/jail": "/srv", + } + dockerfile = open("Dockerfile", "r").read() + jail = None + chroot_dir = "/" + for jail_image in jail_image_to_chroot_dir: + if re.search(r"^FROM %s" % jail_image, dockerfile, re.MULTILINE): + jail = jail_image + chroot_dir = jail_image_to_chroot_dir[jail_image] + break + try: + progress.status("Building image") + image_sha = subprocess.check_output(["docker", "build", "-q", "."], stderr=subprocess.PIPE, shell=False).decode().strip() + + progress.status("Retrieving library paths") + ldd_command = ["-c", "chroot %s /bin/sh -c 'ldd /bin/sh'" % chroot_dir] + ldd_output = subprocess.check_output([ + "docker", + "run", + "--rm", + "--entrypoint", + "/bin/sh", + ] + (["--privileged"] if jail else []) + [ + image_sha, + ] + ldd_command, + stderr=subprocess.PIPE, + shell=False + ).decode() + + libc, ld = None, None + libc_basename, ld_basename = None, None + for lib_path in parse_ldd_output(ldd_output): + if "libc." in lib_path: + libc = lib_path + libc_basename = os.path.basename(lib_path) + if "ld-" in lib_path: + ld = lib_path + ld_basename = os.path.basename(lib_path) + + if not (libc and ld): + progress.failure("Could not find libraries") + return None, None + + progress.status("Copying libraries to current directory") + for filename, basename in zip((libc, ld), (libc_basename, ld_basename)): + cat_command = ["-c", "chroot %s /bin/sh -c '/bin/cat %s'" % (chroot_dir, filename)] + contents = subprocess.check_output([ + "docker", + "run", + "--rm", + "--entrypoint", + "/bin/sh", + ] + (["--privileged"] if jail else []) + [ + image_sha + ] + cat_command, + stderr=subprocess.PIPE, + shell=False + ) + write(basename, contents) + + except subprocess.CalledProcessError as e: + print(e.stderr.decode()) + log.error("docker failed with status: %d" % e.returncode) + + progress.success("Retrieved libraries from Docker image") + return libc_basename, ld_basename + def detect_missing_binaries(args): log.info("Automatically detecting challenge binaries...") # look for challenge binary, libc, and ld in current directory exe, libc, ld = args.exe, args.libc, None + has_dockerfile = False other_files = [] - for filename in os.listdir(): + for filename in os.listdir("."): if not os.path.isfile(filename): continue if not libc and ('libc-' in filename or 'libc.' in filename): libc = filename elif not ld and 'ld-' in filename: ld = filename + elif filename == "Dockerfile": + has_dockerfile = True else: if os.access(filename, os.X_OK): other_files.append(filename) @@ -52,6 +137,9 @@ def detect_missing_binaries(args): exe = other_files[0] elif len(other_files) > 1: log.warning("Failed to find challenge binary. There are multiple binaries in the current directory: %s", other_files) + + if has_dockerfile and exe and not (libc or ld): + libc, ld = get_docker_image_libraries() if exe != args.exe: log.success("Found challenge binary %r", exe) @@ -122,5 +210,5 @@ def main(args): except OSError: pass if __name__ == '__main__': - pwnlib.commandline.common.main(__file__) + pwnlib.commandline.common.main(__file__, main) diff --git a/pwnlib/commandline/unhex.py b/pwnlib/commandline/unhex.py index a254e6b3f..99dad8efd 100644 --- a/pwnlib/commandline/unhex.py +++ b/pwnlib/commandline/unhex.py @@ -30,4 +30,4 @@ def main(args): raise if __name__ == '__main__': - common.main(__file__) + common.main(__file__, main) diff --git a/pwnlib/commandline/update.py b/pwnlib/commandline/update.py index 38ef19a76..2670c0c31 100644 --- a/pwnlib/commandline/update.py +++ b/pwnlib/commandline/update.py @@ -30,4 +30,4 @@ def main(a): subprocess.check_call(result, shell=False) if __name__ == '__main__': - pwnlib.commandline.common.main(__file__) + pwnlib.commandline.common.main(__file__, main) diff --git a/pwnlib/commandline/version.py b/pwnlib/commandline/version.py index 9b820160d..ad25a7570 100644 --- a/pwnlib/commandline/version.py +++ b/pwnlib/commandline/version.py @@ -29,4 +29,4 @@ def main(a): log.info("Pwntools v%s" % version) if __name__ == '__main__': - pwnlib.commandline.common.main(__file__) + pwnlib.commandline.common.main(__file__, main) diff --git a/pwnlib/dynelf.py b/pwnlib/dynelf.py index 5c3948937..e22cf3082 100644 --- a/pwnlib/dynelf.py +++ b/pwnlib/dynelf.py @@ -652,7 +652,7 @@ def _dynamic_load_dynelf(self, libname): break if name: - self.status('Skipping %s' % name) + self.status('Skipping %r' % name) cur = leak.field(cur, LinkMap.l_next) else: diff --git a/pwnlib/elf/elf.py b/pwnlib/elf/elf.py index e8246db3b..02668f0a9 100644 --- a/pwnlib/elf/elf.py +++ b/pwnlib/elf/elf.py @@ -358,6 +358,7 @@ def __init__(self, path, checksec=True): self._populate_functions() self._populate_kernel_version() + self._print_checksec = checksec if checksec: self._describe() @@ -713,14 +714,14 @@ def non_writable_segments(self): @property def libs(self): - """Dictionary of {path: address} for every library loaded for this ELF.""" + """Dictionary of ``{path: address}`` for every library loaded for this ELF.""" if self._libs is None: self._populate_libraries() return self._libs @property def maps(self): - """Dictionary of {name: address} for every mapping in this ELF's address space.""" + """Dictionary of ``{name: address}`` for every mapping in this ELF's address space.""" if self._maps is None: self._populate_libraries() return self._maps @@ -730,12 +731,13 @@ def libc(self): """:class:`.ELF`: If this :class:`.ELF` imports any libraries which contain ``'libc[.-]``, and we can determine the appropriate path to it on the local system, returns a new :class:`.ELF` object pertaining to that library. + Prints the `checksec` output of the library if it was printed for the original ELF too. If not found, the value will be :const:`None`. """ for lib in self.libs: if '/libc.' in lib or '/libc-' in lib: - return ELF(lib) + return ELF(lib, self._print_checksec) def _populate_libraries(self): """ diff --git a/pwnlib/encoders/i386/ascii_shellcode.py b/pwnlib/encoders/i386/ascii_shellcode.py index 0b61ca730..993cfbb1d 100644 --- a/pwnlib/encoders/i386/ascii_shellcode.py +++ b/pwnlib/encoders/i386/ascii_shellcode.py @@ -132,7 +132,7 @@ def _get_allocator(self, size, vocab): Examples: >>> context.update(arch='i386', os='linux') - >>> vocab = bytearray(b'!"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~') + >>> vocab = bytearray(b'!"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~') >>> encoders.i386.ascii_shellcode.encode._get_allocator(300, vocab) bytearray(b'TX-!!!!-!_``-t~~~P\\%!!!!%@@@@') """ @@ -178,7 +178,7 @@ def _find_negatives(self, vocab): Examples: >>> context.update(arch='i386', os='linux') - >>> vocab = bytearray(b'!"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~') + >>> vocab = bytearray(b'!"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~') >>> a, b = encoders.i386.ascii_shellcode.encode._find_negatives(vocab) >>> a & b 0 @@ -212,7 +212,7 @@ def _get_subtractions(self, shellcode, vocab): >>> context.update(arch='i386', os='linux') >>> sc = bytearray(b'ABCDEFGHIGKLMNOPQRSTUVXYZ') - >>> vocab = bytearray(b'!"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~') + >>> vocab = bytearray(b'!"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~') >>> encoders.i386.ascii_shellcode.encode._get_subtractions(sc, vocab) bytearray(b'-(!!!-~NNNP-!=;:-f~~~-~~~~P-!!!!-edee-~~~~P-!!!!-eddd-~~~~P-!!!!-egdd-~~~~P-!!!!-eadd-~~~~P-!!!!-eddd-~~~~P') """ @@ -255,7 +255,7 @@ def _calc_subtractions(self, last, target, vocab): Examples: >>> context.update(arch='i386', os='linux') - >>> vocab = bytearray(b'!"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~') + >>> vocab = bytearray(b'!"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~') >>> print(encoders.i386.ascii_shellcode.encode._calc_subtractions(bytearray(b'\x10'*4), bytearray(b'\x11'*4), vocab)) [bytearray(b'!!!!'), bytearray(b'`___'), bytearray(b'~~~~')] >>> print(encoders.i386.ascii_shellcode.encode._calc_subtractions(bytearray(b'\x11\x12\x13\x14'), bytearray(b'\x15\x16\x17\x18'), vocab)) diff --git a/pwnlib/encoders/mips/xor.py b/pwnlib/encoders/mips/xor.py index 6d14f9c33..bd2fcbc83 100644 --- a/pwnlib/encoders/mips/xor.py +++ b/pwnlib/encoders/mips/xor.py @@ -41,7 +41,7 @@ b'\xff\xff\x08\x21', # addi t0,t0,-1 b'\xff\xff\x10\x05', # bltzal t0,14 b'\x82\x82\x08\x28', # slti t0,zero,-32126 - b'\xe2\xff\xfd\x23', # addi sp,ra,-30 + b'\xe0\xff\xfd\x23', # addi sp,ra,-32 b'\x27\x58\x60\x01', # nor t3,t3,zero b'\x21\xc8\xeb\x03', # addu t9,ra,t3 b'\x82\x82\x17\x28', # slti s7,zero,-32126 @@ -72,7 +72,7 @@ b'\x21\x08\xff\xff', # addi t0,t0,-1 b'\x05\x10\xff\xff', # bltzal t0,14 b'\x28\x08\x82\x82', # slti t0,zero,-32126 - b'\x23\xfd\xff\xe2', # addi sp,ra,-30 + b'\x23\xfd\xff\xe0', # addi sp,ra,-32 b'\x01\x60\x58\x27', # nor t3,t3,zero b'\x03\xeb\xc8\x21', # addu t9,ra,t3 b'\x28\x17\x82\x82', # slti s7,zero,-32126 diff --git a/pwnlib/libcdb.py b/pwnlib/libcdb.py index 7e8fb9119..90dde91ce 100644 --- a/pwnlib/libcdb.py +++ b/pwnlib/libcdb.py @@ -72,6 +72,10 @@ def _turbofast_extract_build_id(path): urls = os.environ['DEBUGINFOD_URLS'].split(' ') DEBUGINFOD_SERVERS = urls + DEBUGINFOD_SERVERS +# Allow to override url with a caching proxy in CI +LIBC_RIP_URL = os.environ.get("PWN_LIBCRIP_URL", "https://libc.rip").rstrip("/") +GITLAB_LIBCDB_URL = os.environ.get("PWN_GITLAB_LIBCDB_URL", "https://gitlab.com").rstrip("/") + # Retry failed lookups after some time NEGATIVE_CACHE_EXPIRY = 60 * 60 * 24 * 7 # 1 week @@ -86,7 +90,7 @@ def provider_libcdb(hex_encoded_id, search_type): from six.moves import urllib # Build the URL using the requested hash type - url_base = "https://gitlab.com/libcdb/libcdb/raw/master/hashes/%s/" % search_type + url_base = "{}/libcdb/libcdb/raw/master/hashes/{}/".format(GITLAB_LIBCDB_URL, search_type) url = urllib.parse.urljoin(url_base, hex_encoded_id) data = b"" @@ -111,7 +115,7 @@ def query_libc_rip(params): # Deferred import because it's slow import requests - url = "https://libc.rip/api/find" + url = "{}/api/find".format(LIBC_RIP_URL) try: result = requests.post(url, json=params, timeout=20) result.raise_for_status() @@ -143,6 +147,7 @@ def provider_libc_rip(search_target, search_type): url = libc_match[0]['download_url'] log.debug("Downloading data from libc.rip: %s", url) + url = url.replace("https://libc.rip", LIBC_RIP_URL) data = wget(url, timeout=20) if not data: @@ -531,7 +536,9 @@ def _find_libc_package_lib_url(libc): libc_match = query_libc_rip({'buildid': enhex(libc.buildid)}) if libc_match is not None: for match in libc_match: - yield match['libs_url'] + # Allow to override url with a caching proxy in CI + ubuntu_archive_url = os.environ.get('PWN_UBUNTU_ARCHIVE_URL', 'http://archive.ubuntu.com').rstrip('/') + yield match['libs_url'].replace('http://archive.ubuntu.com', ubuntu_archive_url) # Check launchpad.net if it's an Ubuntu libc # GNU C Library (Ubuntu GLIBC 2.36-0ubuntu4) diff --git a/pwnlib/qemu.py b/pwnlib/qemu.py index 0ac957c97..ee9fe218e 100644 --- a/pwnlib/qemu.py +++ b/pwnlib/qemu.py @@ -139,7 +139,7 @@ def user_path(): def ld_prefix(path=None, env=None): """Returns the linker prefix for the selected qemu-user binary - >>> pwnlib.qemu.ld_prefix(arch='arm') + >>> pwnlib.qemu.ld_prefix(arch='arm') # doctest: +SKIP '/etc/qemu-binfmt/arm' """ if context.os == 'baremetal': diff --git a/pwnlib/tubes/remote.py b/pwnlib/tubes/remote.py index b84d1d5a5..e9cc82ea0 100644 --- a/pwnlib/tubes/remote.py +++ b/pwnlib/tubes/remote.py @@ -89,6 +89,8 @@ def __init__(self, host, port, import ssl as _ssl ssl_args = ssl_args or {} + if "server_hostname" in ssl_args and sni: + log.error("sni and server_hostname cannot be set at the same time") ssl_context = ssl_context or _ssl.SSLContext(_ssl.PROTOCOL_TLSv1_2) if isinstance(sni, str): ssl_args["server_hostname"] = sni diff --git a/pwnlib/util/lists.py b/pwnlib/util/lists.py index 4d200f8e7..ada0c44f7 100644 --- a/pwnlib/util/lists.py +++ b/pwnlib/util/lists.py @@ -25,8 +25,8 @@ def partition(lst, f, save_keys = False): >>> partition([1,2,3,4,5], lambda x: x&1) [[1, 3, 5], [2, 4]] - >>> partition([1,2,3,4,5], lambda x: x%3, save_keys=True) - OrderedDict([(1, [1, 4]), (2, [2, 5]), (0, [3])]) + >>> partition([1,2,3,4,5], lambda x: x%3, save_keys=True) == collections.OrderedDict([(1, [1, 4]), (2, [2, 5]), (0, [3])]) + True """ d = collections.OrderedDict() diff --git a/pwnlib/util/misc.py b/pwnlib/util/misc.py index d69873572..e55465d33 100644 --- a/pwnlib/util/misc.py +++ b/pwnlib/util/misc.py @@ -370,13 +370,13 @@ def run_in_new_terminal(command, terminal=None, args=None, kill_at_exit=True, pr terminal = 'cmd.exe' args = ['/c', 'start'] distro_name = os.getenv('WSL_DISTRO_NAME') + current_dir = os.getcwd() # Split pane in Windows Terminal if 'WT_SESSION' in os.environ and which('wt.exe'): - args.extend(['wt.exe', '-w', '0', 'split-pane', '-d', '.']) - + args.extend(['wt.exe', '-w', '0', 'split-pane']) if distro_name: - args.extend(['wsl.exe', '-d', distro_name, 'bash', '-c']) + args.extend(['wsl.exe', '-d', distro_name, '--cd', current_dir, 'bash', '-c']) else: args.extend(['bash.exe', '-c']) @@ -504,7 +504,7 @@ def kill(): if terminal == 'qdbus': os.kill(pid, signal.SIGHUP) elif terminal == 'kitty': - subprocess.Popen(["kitten", "@", "close-window", "--match", "id:{}".format(kittyid)]) + subprocess.Popen(["kitten", "@", "close-window", "--match", "id:{}".format(kittyid)], stderr=stderr) else: os.kill(pid, signal.SIGTERM) except OSError: diff --git a/pwnlib/util/safeeval.py b/pwnlib/util/safeeval.py index 6c4a7c5a3..35694976d 100644 --- a/pwnlib/util/safeeval.py +++ b/pwnlib/util/safeeval.py @@ -29,8 +29,8 @@ def _get_opcodes(codeobj): Extract the actual opcodes as a list from a code object >>> c = compile("[1 + 2, (1,2)]", "", "eval") - >>> _get_opcodes(c) - [100, 100, 103, 83] + >>> _get_opcodes(c) # doctest: +ELLIPSIS + [...100, 100, 103, 83] """ import dis if hasattr(dis, 'get_instructions'): diff --git a/travis/libcdb_nginx_cache.conf b/travis/libcdb_nginx_cache.conf new file mode 100644 index 000000000..689784978 --- /dev/null +++ b/travis/libcdb_nginx_cache.conf @@ -0,0 +1,69 @@ +events { + worker_connections 1024; +} + +http { + proxy_cache_path /var/cache/nginx keys_zone=my_cache:1m max_size=1g inactive=12w use_temp_path=off; + log_format cache_st '$remote_addr - $remote_user - $upstream_cache_status [$time_local] ' + '"$request" $status $body_bytes_sent ' + '"$http_referer" "$http_user_agent"'; + access_log /dev/stdout cache_st; + + server { + listen 3000; + proxy_cache my_cache; + + location / { + proxy_set_header Host debuginfod.elfutils.org; + proxy_cache_revalidate on; + proxy_cache_key $scheme://$host$uri$is_args$query_string; + proxy_cache_valid 200 404 12w; + proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504 http_429; + proxy_pass https://debuginfod.elfutils.org/; + } + } + + server { + listen 3001; + proxy_cache my_cache; + + location / { + proxy_set_header Host libc.rip; + proxy_cache_methods GET HEAD POST; + proxy_cache_revalidate on; + proxy_cache_key $scheme://$host$uri$is_args$query_string$request_body; + proxy_cache_valid 200 404 12w; + proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504 http_429; + proxy_pass https://libc.rip/; + } + } + + server { + listen 3002; + proxy_cache my_cache; + + location / { + proxy_set_header Host archive.ubuntu.com; + proxy_cache_revalidate on; + proxy_cache_key $scheme://$host$uri$is_args$query_string; + proxy_cache_valid 200 404 12w; + proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504 http_429; + proxy_pass http://archive.ubuntu.com/; + } + } + + server { + listen 3003; + proxy_cache my_cache; + + location / { + proxy_set_header Host gitlab.com; + proxy_ssl_server_name on; + proxy_cache_revalidate on; + proxy_cache_key $scheme://$host$uri$is_args$query_string; + proxy_cache_valid 200 404 12w; + proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504 http_429; + proxy_pass https://gitlab.com/; + } + } +} diff --git a/travis/setup_avd_fast.sh b/travis/setup_avd_fast.sh index 01cd9c4c0..c177fa934 100644 --- a/travis/setup_avd_fast.sh +++ b/travis/setup_avd_fast.sh @@ -9,16 +9,18 @@ set -ex # - x86 # - x86_64 ANDROID_ABI='x86_64' -ANDROIDV=android-24 +ANDROIDV=android-34 +export ANDROID_AVD_HOME="$HOME/.android/avd" +mkdir -p "$ANDROID_AVD_HOME" # Create our emulator Android Virtual Device (AVD) # --snapshot flag is deprecated, see bitrise-steplib/steps-create-android-emulator#18 -export PATH=$PATH:"$ANDROID_HOME"/cmdline-tools/latest/bin:"$ANDROID_HOME"/platform-tools -yes | sdkmanager --sdk_root="$ANDROID_HOME" --install "system-images;$ANDROIDV;default;$ANDROID_ABI" "emulator" "platform-tools" "platforms;$ANDROIDV" +export PATH=$PATH:"$ANDROID_HOME"/cmdline-tools/latest/bin:"$ANDROID_HOME"/platform-tools:"$ANDROID_HOME"/emulator +yes | sdkmanager --sdk_root="$ANDROID_HOME" --install "system-images;$ANDROIDV;default;$ANDROID_ABI" "emulator" "platform-tools" # "platforms;$ANDROIDV" yes | sdkmanager --sdk_root="$ANDROID_HOME" --licenses -echo no | avdmanager --silent create avd --name android-$ANDROID_ABI --force --package "system-images;$ANDROIDV;default;$ANDROID_ABI" -"$ANDROID_HOME"/emulator/emulator -avd android-$ANDROID_ABI -no-window -no-boot-anim -read-only -no-audio -no-window -no-snapshot -gpu off -accel off & +echo no | avdmanager --verbose create avd --name android-$ANDROID_ABI --force --abi "default/$ANDROID_ABI" --package "system-images;$ANDROIDV;default;$ANDROID_ABI" +emulator -avd android-$ANDROID_ABI -no-window -no-boot-anim -read-only -no-audio -no-window -no-snapshot -gpu off -accel off -no-metrics & adb wait-for-device adb shell id adb shell getprop