From d2253110bd44f026be367e91b0c93dc86132c5d8 Mon Sep 17 00:00:00 2001 From: peace-maker Date: Thu, 24 Oct 2024 15:59:24 +0200 Subject: [PATCH] Add caching proxy for libcdb debuginfod files to CI (#2487) --- .github/workflows/ci.yml | 46 ++++++++++++++++++++++- pwnlib/dynelf.py | 2 +- pwnlib/libcdb.py | 13 +++++-- travis/libcdb_nginx_cache.conf | 69 ++++++++++++++++++++++++++++++++++ 4 files changed, 124 insertions(+), 6 deletions(-) create mode 100644 travis/libcdb_nginx_cache.conf diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5533a9093..23d0c2309 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,21 @@ jobs: os: ubuntu-22.04 runs-on: ${{ matrix.os }} timeout-minutes: 30 + services: + libcdb-cache: + image: nginx + volumes: + - /home/runner/libcdb-cache:/var/cache/nginx + ports: + - 3000:3000 # https://debuginfod.elfutils.org proxy cache + - 3001:3001 # https://libc.rip/ proxy cache + - 3002:3002 # http://archive.ubuntu.com/ proxy cache + - 3003:3003 # https://gitlab.com/ proxy cache + env: + DEBUGINFOD_URLS: http://localhost:3000/ + PWN_LIBCRIP_URL: http://localhost:3001/ + PWN_UBUNTU_ARCHIVE_URL: http://localhost:3002/ + PWN_GITLAB_LIBCDB_URL: http://localhost:3003/ steps: - uses: actions/checkout@v4 with: @@ -21,6 +36,28 @@ jobs: run: | git fetch origin git log --oneline --graph -10 + + - name: Fix libcdb-cache permissions + id: fix-perms + run: | + sudo chown -R runner:runner /home/runner/libcdb-cache + echo "date=$(/bin/date -u "+%Y%m%d%H%M%S")" >> $GITHUB_OUTPUT + + - name: Cache for libcdb requests + uses: actions/cache@v4 + with: + path: ~/libcdb-cache + key: libcdb-python${{ matrix.python_version }}-${{ steps.fix-perms.outputs.date }} + restore-keys: | + libcdb-python${{ matrix.python_version }}- + libcdb- + + - name: Install libcdb-cache service config + run: | + sudo chown -R 101:101 /home/runner/libcdb-cache + container_id=$(docker ps --all --filter volume=/home/runner/libcdb-cache --no-trunc --format "{{.ID}}") + docker cp ./travis/libcdb_nginx_cache.conf $container_id:/etc/nginx/nginx.conf + docker restart $container_id - name: Install RPyC for gdb run: | @@ -29,11 +66,10 @@ jobs: sudo apt-get update && sudo apt-get install -y python3-pip gdb gdbserver /usr/bin/python -m pip install --break-system-packages rpyc || /usr/bin/python -m pip install rpyc gdb --batch --quiet --nx --nh --ex 'py import rpyc; print(rpyc.version.version)' - + - name: Cache for pip uses: actions/cache@v4 if: matrix.python_version == '2.7' - id: cache-pip with: path: ~/.cache/pip key: ${{ matrix.os }}-${{ matrix.python_version }}-cache-pip-${{ hashFiles('**/pyproject.toml', '**/requirements*.txt') }} @@ -224,6 +260,12 @@ jobs: name: coverage-${{ matrix.python_version }} path: .coverage* include-hidden-files: true + + - name: Fix libcdb-cache permissions + run: | + container_id=$(docker ps --filter volume=/home/runner/libcdb-cache --no-trunc --format "{{.ID}}") + docker stop $container_id + sudo chown -R runner:runner /home/runner/libcdb-cache windows-test: runs-on: windows-latest diff --git a/pwnlib/dynelf.py b/pwnlib/dynelf.py index 5c3948937..e22cf3082 100644 --- a/pwnlib/dynelf.py +++ b/pwnlib/dynelf.py @@ -652,7 +652,7 @@ def _dynamic_load_dynelf(self, libname): break if name: - self.status('Skipping %s' % name) + self.status('Skipping %r' % name) cur = leak.field(cur, LinkMap.l_next) else: diff --git a/pwnlib/libcdb.py b/pwnlib/libcdb.py index f0c21ccb4..d8a4325bf 100644 --- a/pwnlib/libcdb.py +++ b/pwnlib/libcdb.py @@ -72,6 +72,10 @@ def _turbofast_extract_build_id(path): urls = os.environ['DEBUGINFOD_URLS'].split(' ') DEBUGINFOD_SERVERS = urls + DEBUGINFOD_SERVERS +# Allow to override url with a caching proxy in CI +LIBC_RIP_URL = os.environ.get("PWN_LIBCRIP_URL", "https://libc.rip").rstrip("/") +GITLAB_LIBCDB_URL = os.environ.get("PWN_GITLAB_LIBCDB_URL", "https://gitlab.com").rstrip("/") + # Retry failed lookups after some time NEGATIVE_CACHE_EXPIRY = 60 * 60 * 24 * 7 # 1 week @@ -86,7 +90,7 @@ def provider_libcdb(hex_encoded_id, search_type): from six.moves import urllib # Build the URL using the requested hash type - url_base = "https://gitlab.com/libcdb/libcdb/raw/master/hashes/%s/" % search_type + url_base = "{}/libcdb/libcdb/raw/master/hashes/{}/".format(GITLAB_LIBCDB_URL, search_type) url = urllib.parse.urljoin(url_base, hex_encoded_id) data = b"" @@ -111,7 +115,7 @@ def query_libc_rip(params): # Deferred import because it's slow import requests - url = "https://libc.rip/api/find" + url = "{}/api/find".format(LIBC_RIP_URL) try: result = requests.post(url, json=params, timeout=20) result.raise_for_status() @@ -143,6 +147,7 @@ def provider_libc_rip(search_target, search_type): url = libc_match[0]['download_url'] log.debug("Downloading data from libc.rip: %s", url) + url = url.replace("https://libc.rip", LIBC_RIP_URL) data = wget(url, timeout=20) if not data: @@ -529,7 +534,9 @@ def _find_libc_package_lib_url(libc): libc_match = query_libc_rip({'buildid': enhex(libc.buildid)}) if libc_match is not None: for match in libc_match: - yield match['libs_url'] + # Allow to override url with a caching proxy in CI + ubuntu_archive_url = os.environ.get('PWN_UBUNTU_ARCHIVE_URL', 'http://archive.ubuntu.com').rstrip('/') + yield match['libs_url'].replace('http://archive.ubuntu.com', ubuntu_archive_url) # Check launchpad.net if it's an Ubuntu libc # GNU C Library (Ubuntu GLIBC 2.36-0ubuntu4) diff --git a/travis/libcdb_nginx_cache.conf b/travis/libcdb_nginx_cache.conf new file mode 100644 index 000000000..689784978 --- /dev/null +++ b/travis/libcdb_nginx_cache.conf @@ -0,0 +1,69 @@ +events { + worker_connections 1024; +} + +http { + proxy_cache_path /var/cache/nginx keys_zone=my_cache:1m max_size=1g inactive=12w use_temp_path=off; + log_format cache_st '$remote_addr - $remote_user - $upstream_cache_status [$time_local] ' + '"$request" $status $body_bytes_sent ' + '"$http_referer" "$http_user_agent"'; + access_log /dev/stdout cache_st; + + server { + listen 3000; + proxy_cache my_cache; + + location / { + proxy_set_header Host debuginfod.elfutils.org; + proxy_cache_revalidate on; + proxy_cache_key $scheme://$host$uri$is_args$query_string; + proxy_cache_valid 200 404 12w; + proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504 http_429; + proxy_pass https://debuginfod.elfutils.org/; + } + } + + server { + listen 3001; + proxy_cache my_cache; + + location / { + proxy_set_header Host libc.rip; + proxy_cache_methods GET HEAD POST; + proxy_cache_revalidate on; + proxy_cache_key $scheme://$host$uri$is_args$query_string$request_body; + proxy_cache_valid 200 404 12w; + proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504 http_429; + proxy_pass https://libc.rip/; + } + } + + server { + listen 3002; + proxy_cache my_cache; + + location / { + proxy_set_header Host archive.ubuntu.com; + proxy_cache_revalidate on; + proxy_cache_key $scheme://$host$uri$is_args$query_string; + proxy_cache_valid 200 404 12w; + proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504 http_429; + proxy_pass http://archive.ubuntu.com/; + } + } + + server { + listen 3003; + proxy_cache my_cache; + + location / { + proxy_set_header Host gitlab.com; + proxy_ssl_server_name on; + proxy_cache_revalidate on; + proxy_cache_key $scheme://$host$uri$is_args$query_string; + proxy_cache_valid 200 404 12w; + proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504 http_429; + proxy_pass https://gitlab.com/; + } + } +}