Merge branch 'dev' into feat/libcdb-cli

.github/ISSUE_TEMPLATE/bug_report.md

@@ -39,7 +39,7 @@ You should see `[DEBUG]` statements that show what's happening behind the scenes
 
 ## Verify on Ubuntu
 
-If possible, please verify that your issue occurs on 64-bit Ubuntu 18.04.  We provide a Dockerfile based on Ubuntu 18.04 via `docker.io` to make this super simple, no VM required!
+If possible, please verify that your issue occurs on 64-bit Ubuntu 22.04.  We provide a Dockerfile based on Ubuntu 22.04 via `docker.io` to make this super simple, no VM required!
 
 ```sh
 # Download the Docker image

.github/workflows/android.yml

@@ -6,24 +6,21 @@ jobs:
   android-test:
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: ['3.10']
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     timeout-minutes: 30
     steps:
     - uses: actions/checkout@v4
 
-    - name: Cache for pip
-      uses: actions/cache@v4
-      id: cache-pip
-      with:
-        path: ~/.cache/pip
-        key: ${{ matrix.os }}-cache-pip
-
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+        cache-dependency-path: |
+          **/pyproject.toml
+          **/requirements*.txt
 
     - name: Install Linux dependencies
       run: |
@@ -35,6 +32,19 @@ jobs:
           binutils-arm-linux-gnueabihf \
           libc6-dbg
 
+    - name: Cache for avd
+      uses: actions/cache@v4
+      id: cache-avd
+      with:
+        path: |
+          ~/.android
+          /usr/local/lib/android/sdk/emulator
+          /usr/local/lib/android/sdk/platform-tools
+          /usr/local/lib/android/sdk/system-images
+        key: ${{ matrix.os }}-cache-avd-${{ hashFiles('travis/setup_avd*.sh') }}
+        restore-keys: |
+          ${{ matrix.os }}-cache-avd-
+
     - name: Install Android AVD
       run: |
         sudo usermod -aG kvm $USER

.github/workflows/ci.yml

@@ -5,10 +5,28 @@ jobs:
   test:
     strategy:
       matrix:
-        python_version: ['2.7', '3.10']
+        python_version: ['3.10', '3.12']
         os: [ubuntu-latest]
+        include:
+          - python_version: '2.7'
+            os: ubuntu-22.04
     runs-on: ${{ matrix.os }}
     timeout-minutes: 30
+    services:
+      libcdb-cache:
+        image: nginx
+        volumes:
+          - /home/runner/libcdb-cache:/var/cache/nginx
+        ports:
+          - 3000:3000  # https://debuginfod.elfutils.org proxy cache
+          - 3001:3001  # https://libc.rip/ proxy cache
+          - 3002:3002  # http://archive.ubuntu.com/ proxy cache
+          - 3003:3003  # https://gitlab.com/ proxy cache
+    env:
+      DEBUGINFOD_URLS: http://localhost:3000/
+      PWN_LIBCRIP_URL: http://localhost:3001/
+      PWN_UBUNTU_ARCHIVE_URL: http://localhost:3002/
+      PWN_GITLAB_LIBCDB_URL:  http://localhost:3003/
     steps:
     - uses: actions/checkout@v4
       with:
@@ -18,26 +36,54 @@ jobs:
       run: |
         git fetch origin
         git log --oneline --graph -10
+    
+    - name: Fix libcdb-cache permissions
+      id: fix-perms
+      run: |
+        sudo chown -R runner:runner /home/runner/libcdb-cache
+        echo "date=$(/bin/date -u "+%Y%m%d%H%M%S")" >> $GITHUB_OUTPUT
+
+    - name: Cache for libcdb requests
+      uses: actions/cache@v4
+      with:
+        path: ~/libcdb-cache
+        key: libcdb-python${{ matrix.python_version }}-${{ steps.fix-perms.outputs.date }}
+        restore-keys: |
+          libcdb-python${{ matrix.python_version }}-
+          libcdb-
+
+    - name: Install libcdb-cache service config
+      run: |
+        sudo chown -R 101:101 /home/runner/libcdb-cache
+        container_id=$(docker ps --all --filter volume=/home/runner/libcdb-cache --no-trunc --format "{{.ID}}")
+        docker cp ./travis/libcdb_nginx_cache.conf $container_id:/etc/nginx/nginx.conf
+        docker restart $container_id
 
     - name: Install RPyC for gdb
       run: |
-        # The version packaged in python3-rpyc is too old on Ubuntu 22.04
+        # The version packaged in python3-rpyc is too old on Ubuntu 24.04
+        # We use ^6.0 from pip.
         sudo apt-get update && sudo apt-get install -y python3-pip gdb gdbserver
-        /usr/bin/python -m pip install rpyc
+        /usr/bin/python -m pip install --break-system-packages rpyc || /usr/bin/python -m pip install rpyc
         gdb --batch --quiet --nx --nh --ex 'py import rpyc; print(rpyc.version.version)'
-
+    
     - name: Cache for pip
       uses: actions/cache@v4
-      id: cache-pip
+      if: matrix.python_version == '2.7'
       with:
         path: ~/.cache/pip
-        key: ${{ matrix.os }}-cache-pip
+        key: ${{ matrix.os }}-${{ matrix.python_version }}-cache-pip-${{ hashFiles('**/pyproject.toml', '**/requirements*.txt') }}
+        restore-keys: ${{ matrix.os }}-${{ matrix.python_version }}-cache-pip-
 
     - name: Set up Python ${{ matrix.python_version }}
       if: matrix.python_version != '2.7'
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python_version }}
+        cache: 'pip'
+        cache-dependency-path: |
+          **/pyproject.toml
+          **/requirements*.txt
 
     - name: Set up Python 2.7
       if: matrix.python_version == '2.7'
@@ -203,7 +249,7 @@ jobs:
         python -m build
 
     - uses: actions/upload-artifact@v4
-      if: matrix.python_version != '2.7'
+      if: matrix.python_version == '3.10'
       with:
         name: packages
         path: dist/
@@ -214,6 +260,12 @@ jobs:
         name: coverage-${{ matrix.python_version }}
         path: .coverage*
         include-hidden-files: true
+
+    - name: Fix libcdb-cache permissions
+      run: |
+        container_id=$(docker ps --filter volume=/home/runner/libcdb-cache --no-trunc --format "{{.ID}}")
+        docker stop $container_id
+        sudo chown -R runner:runner /home/runner/libcdb-cache
 
   windows-test:
     runs-on: windows-latest
@@ -252,7 +304,7 @@ jobs:
 
     - name: Install coveralls
       run: |
-        pip install tomli coveralls
+        pip install --break-system-packages tomli coveralls
 
     - name: Upload coverage to coveralls.io
       run: |

.github/workflows/lint.yml

@@ -5,23 +5,21 @@ jobs:
   lint:
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: ['3.10']
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     timeout-minutes: 30
     steps:
     - uses: actions/checkout@v4
-    - name: Cache for pip
-      uses: actions/cache@v4
-      id: cache-pip
-      with:
-        path: ~/.cache/pip
-        key: ${{ matrix.os }}-cache-pip
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+        cache-dependency-path: |
+          **/pyproject.toml
+          **/requirements*.txt
 
     - name: Critical lint
       run: |

.github/workflows/pylint.yml

@@ -5,23 +5,21 @@ jobs:
   build:
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: ['3.10']
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     timeout-minutes: 30
     steps:
     - uses: actions/checkout@v4
-    - name: Cache for pip
-      uses: actions/cache@v4
-      id: cache-pip
-      with:
-        path: ~/.cache/pip
-        key: ${{ matrix.os }}-cache-pip
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+        cache-dependency-path: |
+          **/pyproject.toml
+          **/requirements*.txt
 
     - name: PyLint
       run: |

CHANGELOG.md

@@ -79,6 +79,9 @@ The table below shows which release corresponds to each branch, and what date th
 - [#2444][2444] Add `ELF.close()` to release resources
 - [#2413][2413] libcdb: improve the search speed of `search_by_symbol_offsets` in local libc-database
 - [#2470][2470] Fix waiting for gdb under WSL2
+- [#2479][2479] Support extracting libraries from Docker image in `pwn template`
+- [#2483][2483] Only print `checksec` output of `ELF.libc` when it was printed for the `ELF` already
+- [#2482][2482] Throw error when using `sni` and setting `server_hostname` manually in `remote`
 - [#2478][2478] libcdb-cli: add `--offline-only`, refactor unstrip and add fetch parser for download libc-database
 
 [2471]: https://github.com/Gallopsled/pwntools/pull/2471
@@ -87,6 +90,9 @@ The table below shows which release corresponds to each branch, and what date th
 [2444]: https://github.com/Gallopsled/pwntools/pull/2444
 [2413]: https://github.com/Gallopsled/pwntools/pull/2413
 [2470]: https://github.com/Gallopsled/pwntools/pull/2470
+[2479]: https://github.com/Gallopsled/pwntools/pull/2479
+[2483]: https://github.com/Gallopsled/pwntools/pull/2483
+[2482]: https://github.com/Gallopsled/pwntools/pull/2482
 [2478]: https://github.com/Gallopsled/pwntools/pull/2478
 
 ## 4.14.0 (`beta`)

README.md

@@ -32,7 +32,7 @@ To get you started, we've provided some example solutions for past CTF challenge
 
 # Installation
 
-Pwntools is best supported on 64-bit Ubuntu LTS releases (14.04, 16.04, 18.04, and 20.04).  Most functionality should work on any Posix-like distribution (Debian, Arch, FreeBSD, OSX, etc.).  
+Pwntools is best supported on 64-bit Ubuntu LTS releases (18.04, 20.04, 22.04, and 24.04).  Most functionality should work on any Posix-like distribution (Debian, Arch, FreeBSD, OSX, etc.).  
 
 Python3 is suggested, but Pwntools still works with Python 2.7.  Most of the functionality of pwntools is self-contained and Python-only.  You should be able to get running quickly with
 

pwnlib/adb/adb.py

@@ -123,7 +123,7 @@ def current_device(any=False):
 
         >>> device = adb.current_device(any=True)
         >>> device  # doctest: +ELLIPSIS
-        AdbDevice(serial='emulator-5554', type='device', port='emulator', product='sdk_...phone_...', model='...', device='generic...')
+        AdbDevice(serial='emulator-5554', type='device', port='emulator', product='sdk_...phone..._...', model='...', device='...')
         >>> device.port
         'emulator'
     """
@@ -259,7 +259,7 @@ class AdbDevice(Device):
         >>> device.os
         'android'
         >>> device.product  # doctest: +ELLIPSIS
-        'sdk_...phone_...'
+        'sdk_...phone..._...'
         >>> device.serial
         'emulator-5554'
     """
@@ -880,7 +880,7 @@ def which(name, all = False, *a, **kw):
         >>> adb.which('sh')
         '/system/bin/sh'
         >>> adb.which('sh', all=True)
-        ['/system/bin/sh']
+        ['/system/bin/sh', '/vendor/bin/sh']
 
         >>> adb.which('foobar') is None
         True
@@ -988,7 +988,7 @@ def proc_exe(pid):
        :skipif: skip_android
 
         >>> adb.proc_exe(1)
-        b'/init'
+        b'/system/bin/init'
     """
     with context.quiet:
         io  = process(['realpath','/proc/%d/exe' % pid])
@@ -1365,7 +1365,7 @@ def compile(source):
         >>> filename = adb.compile(temp)
         >>> sent = adb.push(filename, "/data/local/tmp")
         >>> adb.process(sent).recvall() # doctest: +ELLIPSIS
-        b'... /system/lib64/libc.so\n...'
+        b'... /system/lib64/libc++.so\n...'
     """
 
     ndk_build = misc.which('ndk-build')

pwnlib/asm.py

@@ -908,8 +908,8 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
            0:   b8 17 00 00 00          mov    eax, 0x17
         >>> print(disasm(unhex('48c7c017000000'), arch = 'amd64'))
            0:   48 c7 c0 17 00 00 00    mov    rax, 0x17
-        >>> print(disasm(unhex('04001fe552009000'), arch = 'arm'))
-           0:   e51f0004        ldr     r0, [pc, #-4]   ; 0x4
+        >>> print(disasm(unhex('04001fe552009000'), arch = 'arm'))  # doctest: +ELLIPSIS
+           0:   e51f0004        ldr     r0, [pc, #-4]   ...
            4:   00900052        addseq  r0, r0, r2, asr r0
         >>> print(disasm(unhex('4ff00500'), arch = 'thumb', bits=32))
            0:   f04f 0005       mov.w   r0, #5

pwnlib/commandline/asm.py

@@ -137,4 +137,4 @@ def main(args):
         args.output.write(b'\n')
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/checksec.py

@@ -41,4 +41,4 @@ def main(args):
             print("{name}: {error}".format(name=f.name, error=e))
 
 if __name__ == '__main__':
-    common.main(__file__)
+    common.main(__file__, main)

pwnlib/commandline/common.py

@@ -25,8 +25,18 @@ def context_arg(arg):
                                  prog='pwn')
 parser_commands = parser.add_subparsers(dest='command')
 
-def main(file=sys.argv[0]):
-    import pwnlib.commandline.main
+def main(file=sys.argv[0], command_main=None):
     name = os.path.splitext(os.path.basename(file))[0]
+    if command_main is None:
+        import importlib
+        command_main = importlib.import_module('pwnlib.commandline.%s' % name).main
     sys.argv.insert(1, name)
-    pwnlib.commandline.main.main()
+    entrypoint({name: command_main})
+
+def entrypoint(commands):
+    if len(sys.argv) < 2:
+        parser.print_usage()
+        sys.exit()
+    args = parser.parse_args()
+    with context.local(log_console = sys.stderr):
+        commands[args.command](args)

pwnlib/commandline/constgrep.py

@@ -133,4 +133,4 @@ def main(args):
                 print('(%s) == %s' % (' | '.join(k for v, k in good), args.constant))
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/cyclic.py

@@ -107,4 +107,4 @@ def main(args):
             out.write(b'\n')
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/debug.py

@@ -102,4 +102,4 @@ def main(args):
         gdb.debug(target, gdbscript=gdbscript, sysroot=args.sysroot).interactive()
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/disablenx.py

@@ -24,4 +24,4 @@ def main(args):
         ELF(e.path)
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/disasm.py

@@ -110,4 +110,4 @@ def main(args):
     print(disasm(dat, vma=safeeval.const(args.address)))
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/elfdiff.py

@@ -59,4 +59,4 @@ def main(a):
     print(diff(x, y))
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/elfpatch.py

@@ -34,4 +34,4 @@ def main(a):
     getattr(sys.stdout, 'buffer', sys.stdout).write(elf.get_data())
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)