Merge pull request #4 from Flared/isra/flared-8.1.1

Flared 8.1.1

.github/ISSUE_TEMPLATE/config.yml

@@ -1,6 +1,5 @@
 blank_issues_enabled: false
 contact_links:
     - name: Support
-      url: https://stackoverflow.com/questions/tagged/mitmproxy
-      about: Please do not use GitHub for support requests.
-             If you have questions on how to use mitmproxy, please ask them on StackOverflow!
+      url: https://github.com/mitmproxy/mitmproxy/discussions
+      about: If you have questions on how to use mitmproxy, ask them on the discussions page!

.github/workflows/main.yml

@@ -24,7 +24,7 @@ jobs:
           persist-credentials: false
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.9'
+          python-version: '3.10'
       - run: pip install tox
       - run: tox -e flake8
   filename-matching:
@@ -35,7 +35,7 @@ jobs:
           persist-credentials: false
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.9'
+          python-version: '3.10'
       - run: pip install tox
       - run: tox -e filename_matching
   mypy:
@@ -46,7 +46,7 @@ jobs:
           persist-credentials: false
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.9'
+          python-version: '3.10'
       - run: pip install tox
       - run: tox -e mypy
   individual-coverage:
@@ -58,7 +58,7 @@ jobs:
           fetch-depth: 0
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.9'
+          python-version: '3.9'  # there's a weird bug on 3.10 where some lines are not counted as covered.
       - run: pip install tox
       - run: tox -e individual_coverage
   test:
@@ -67,15 +67,13 @@ jobs:
       matrix:
         include:
           - os: ubuntu-latest
-            py: '3.10.0-rc - 3.10'
+            py: "3.10"
           - os: windows-latest
-            py: 3.9
+            py: "3.10"
           - os: macos-latest
-            py: 3.9
+            py: "3.10"
           - os: ubuntu-latest
             py: 3.9
-          - os: ubuntu-latest
-            py: 3.8
     runs-on: ${{ matrix.os }}
     steps:
       - run: printenv
@@ -88,6 +86,14 @@ jobs:
           python-version: ${{ matrix.py }}
       - run: pip install tox
       - run: tox -e py
+        if: matrix.os != 'ubuntu-latest'
+      - name: Run tox -e py (without internet)
+        run: |
+          # install dependencies (requires internet connectivity)
+          tox -e py --notest  
+          # run tests with loopback only. We need to sudo for unshare, which means we need an absolute path for tox.
+          sudo unshare --net -- sh -c "ip link set lo up; $(which tox) -e py"  
+        if: matrix.os == 'ubuntu-latest'
       - uses: codecov/codecov-action@a1ed4b322b4b38cb846afb5a0ebfa17086917d27
         # mirrored below and at https://github.com/mitmproxy/mitmproxy/settings/actions
         with:
@@ -118,7 +124,7 @@ jobs:
           fetch-depth: 0
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.9'
+          python-version: '3.10'
       - if: matrix.platform == 'windows'
         uses: actions/cache@v2
         with:
@@ -169,10 +175,10 @@ jobs:
           persist-credentials: false
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.9'
+          python-version: '3.10'
       - run: |
-          wget -q https://github.com/gohugoio/hugo/releases/download/v0.88.1/hugo_extended_0.88.1_Linux-64bit.deb
-          echo "865ab9a930e0a9e4957e7dbfdf91c32847324f022e33271b5661d5717600bc2b hugo_extended_0.88.1_Linux-64bit.deb" | sha256sum -c
+          wget -q https://github.com/gohugoio/hugo/releases/download/v0.92.1/hugo_extended_0.92.1_Linux-64bit.deb
+          echo "a9440adfd3ecce40089def287dee4e42ffae252ba08c77d1ac575b880a079ce6 hugo_extended_0.92.1_Linux-64bit.deb" | sha256sum -c
           sudo dpkg -i hugo*.deb
       - run: pip install -e .[dev]
       - run: ./docs/build.py
@@ -205,7 +211,7 @@ jobs:
           persist-credentials: false
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.9'
+          python-version: '3.10'
       - uses: actions/download-artifact@v2
         with:
           name: binaries.linux
@@ -239,7 +245,7 @@ jobs:
           persist-credentials: false
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.9'
+          python-version: '3.10'
       - run: sudo apt-get update
       - run: sudo apt-get install -y twine awscli
       - uses: actions/download-artifact@v2

.gitignore

@@ -18,6 +18,7 @@ mitmproxy/contrib/kaitaistruct/*.ksy
 .pytest_cache
 __pycache__
 .hypothesis/
+.hugo_build.lock
 
 # UI
 

CHANGELOG.md

@@ -2,8 +2,87 @@
 
 ## Unreleased: mitmproxy next
 
+## 28 June 2022: mitmproxy 8.1.1
+
+* Support specifying the local address for outgoing connections
+  ([#5364](https://github.com/mitmproxy/mitmproxy/discussions/5364), @meitinger)
+* Fix a bug where an excess empty chunk has been sent for chunked HEAD request.
+  ([#5372](https://github.com/mitmproxy/mitmproxy/discussions/5372), @jixunmoe)
+* Drop pkg_resources dependency.
+  ([#5401](https://github.com/mitmproxy/mitmproxy/issues/5401), @PavelICS)
+* Fix huge (>65kb) http2 responses corrupted.
+  ([#5428](https://github.com/mitmproxy/mitmproxy/issues/5428), @dhabensky)
+* Remove overambitious assertions in the HTTP state machine,
+  fix some error handling.
+  ([#5383](https://github.com/mitmproxy/mitmproxy/issues/5383), @mhils)
+
+## 15 May 2022: mitmproxy 8.1.0
+
+* DNS support
+  ([#5232](https://github.com/mitmproxy/mitmproxy/pull/5232), @meitinger)
+* Mitmproxy now requires Python 3.9 or above.
+  ([#5233](https://github.com/mitmproxy/mitmproxy/issues/5233), @mhils)
+* Fix a memory leak in mitmdump where flows were kept in memory.
+  ([#4786](https://github.com/mitmproxy/mitmproxy/issues/4786), @mhils)
+* Replayed flows retain their current position in the flow list.
+  ([#5227](https://github.com/mitmproxy/mitmproxy/issues/5227), @mhils)
+* Periodically send HTTP/2 ping frames to keep connections alive.
+  ([#5046](https://github.com/mitmproxy/mitmproxy/issues/5046), @EndUser509)
+* Console Performance Improvements
+  ([#3427](https://github.com/mitmproxy/mitmproxy/issues/3427), @BkPHcgQL3V)
+* Warn users if server side event responses are received without streaming.
+  ([#4469](https://github.com/mitmproxy/mitmproxy/issues/4469), @mhils)
+* Add flatpak support to the browser addon
+  ([#5200](https://github.com/mitmproxy/mitmproxy/issues/5200), @pauloromeira)
+* Add example addon to dump contents to files based on a filter expression 
+  ([#5190](https://github.com/mitmproxy/mitmproxy/issues/5190), @redraw)
+* Fix a bug where the wrong SNI is sent to an upstream HTTPS proxy
+  ([#5109](https://github.com/mitmproxy/mitmproxy/issues/5109), @mhils)
+* Make sure that mitmproxy displays error messages on startup.
+  ([#5225](https://github.com/mitmproxy/mitmproxy/issues/5225), @mhils)
+* Add example addon for domain fronting.
+  ([#5217](https://github.com/mitmproxy/mitmproxy/issues/5217), @randomstuff)
+* Improve cut addon to better handle binary contents
+  ([#3965](https://github.com/mitmproxy/mitmproxy/issues/3965), @mhils)
+* Fix text truncation for full-width characters 
+  ([#4278](https://github.com/mitmproxy/mitmproxy/issues/4278), @kjy00302)
+* Fix mitmweb export copy failed in non-secure domain.
+  ([#5264](https://github.com/mitmproxy/mitmproxy/issues/5264), @Pactortester)
+* Add example script for manipulating cookies.
+  ([#5278](https://github.com/mitmproxy/mitmproxy/issues/5278), @WillahScott)
+* When opening an external viewer for message contents, mailcap files are not considered anymore.  
+  This preempts the upcoming deprecation of Python's `mailcap` module. 
+  ([#5297](https://github.com/mitmproxy/mitmproxy/issues/5297), @KORraNpl)
+* Fix hostname encoding for IDNA domains in upstream mode.
+  ([#5316](https://github.com/mitmproxy/mitmproxy/issues/5316), @nneonneo)
+* Fix hot reloading of contentviews.
+  ([#5319](https://github.com/mitmproxy/mitmproxy/issues/5319), @nneonneo)
+* Ignore HTTP/2 information responses instead of raising an error.
+  ([#5332](https://github.com/mitmproxy/mitmproxy/issues/5332), @mhils)
+* Improve performance and memory usage by reusing OpenSSL contexts.
+  ([#5339](https://github.com/mitmproxy/mitmproxy/issues/5339), @mhils)
+* Fix handling of multiple Cookie headers when proxying HTTP/2 to HTTP/1
+  ([#5337](https://github.com/mitmproxy/mitmproxy/issues/5337), @rinsuki)
+
+## 19 March 2022: mitmproxy 8.0.0
+
+### Major Changes
+
+* Major improvements to the web interface (@gorogoroumaru)
+* Event hooks can now be async (@nneonneo, [#5106](https://github.com/mitmproxy/mitmproxy/issues/5106))
+* New [`tls_{established,failed}_{client,server}` event hooks](https://docs.mitmproxy.org/dev/api/events.html#TLSEvents)
+  to record negotiation success/failure (@mhils, [#4790](https://github.com/mitmproxy/mitmproxy/pull/4790))
+
+### Security Fixes
+
+* [CVE-2022-24766](https://github.com/mitmproxy/mitmproxy/security/advisories/GHSA-gcx2-gvj7-pxv3):
+  Fix request smuggling vulnerability reported by @zeyu2001 (@mhils)
+
+### Full Changelog
+
 * Support proxy authentication for SOCKS v5 mode (@starplanet)
-* fix some responses not being decoded properly if the encoding was uppercase #4735 (@Mattwmaster58)
+* Make it possible to ignore connections in the tls_clienthello event hook (@mhils)
+* fix some responses not being decoded properly if the encoding was uppercase (#4735, @Mattwmaster58)
 * Trigger event hooks for flows with semantically invalid requests, for example invalid content-length headers (@mhils)
 * Improve error message on TLS version mismatch (@mhils)
 * Windows: Switch to Python's default asyncio event loop, which increases the number of sockets
@@ -14,6 +93,28 @@
 * Fix a crash caused when editing string option (#4852, @rbdixon)
 * Base container image bumped to Debian 11 Bullseye (@Kriechi)
 * Upstream replays don't do CONNECT on plaintext HTTP requests (#4876, @HoffmannP)
+* Remove workarounds for old pyOpenSSL versions (#4831, @KarlParkinson)
+* Add fonts to asset filter (~a) (#4928, @elespike)
+* Fix bug that crashed when using `view.flows.resolve` (#4916, @rbdixon)
+* Fix a bug where `running()` is invoked twice on startup (#3584, @mhils)
+* Correct documentation example for User-Agent header modification (#4997, @jamesyale)
+* Fix random connection stalls (#5040, @EndUser509)
+* Add `n` new flow keybind to mitmweb (#5061, @ianklatzco)
+* Fix compatibility with BoringSSL (@pmoulton)
+* Added `WebSocketMessage.injected` flag (@Prinzhorn)
+* Add example addon for saving streamed data to individual files (@EndUser509)
+* Change connection event hooks to be blocking.
+  Processing will only resume once the event hook has finished. (@Prinzhorn)
+* Reintroduce `Flow.live`, which signals if a flow belongs to a currently active connection. (#4207, @mhils)
+* Speculative fix for some rare HTTP/2 connection stalls (#5158, @EndUser509)
+* Add ability to specify custom ports with LDAP authentication (#5068, @demonoidvk)
+* Add support for rotating saved streams every hour or day (@EndUser509)
+* Console Improvements on Windows (@mhils)
+* Fix processing of `--set` options (#5067, @marwinxxii) 
+* Lowercase user-added header names and emit a log message to notify the user when using HTTP/2 (#4746, @mhils)
+* Exit early if there are errors on startup (#4544, @mhils)
+* Fixed encoding guessing: only search for meta tags in HTML bodies (##4566, @Prinzhorn)
+* Binaries are now built with Python 3.10 (@mhils)
 
 ## 28 September 2021: mitmproxy 7.0.4
 

CONTRIBUTING.md

@@ -5,7 +5,7 @@ forward, please consider contributing in the following areas:
 
 - **Maintenance:** We are *incredibly* thankful for individuals who are stepping up and helping with maintenance. This
   includes (but is not limited to) triaging issues, reviewing pull requests and picking up stale ones, helping out other
-  users on [StackOverflow](https://stackoverflow.com/questions/tagged/mitmproxy), creating minimal, complete and
+  users on [GitHub Discussions](https://github.com/mitmproxy/mitmproxy/discussions), creating minimal, complete and
   verifiable examples or test cases for existing bug reports, updating documentation, or fixing minor bugs that have
   recently been reported.
 - **Code Contributions:** We actively mark issues that we consider are [good first contributions](
@@ -14,7 +14,7 @@ forward, please consider contributing in the following areas:
 
 ## Development Setup
 
-To get started hacking on mitmproxy, please install a recent version of Python (we require at least Python 3.8).
+To get started hacking on mitmproxy, please install a recent version of Python (we require at least Python 3.9).
 Then, do the following:
 
 ##### Linux / macOS

README.md

@@ -29,9 +29,9 @@ The documentation for mitmproxy is available on our website:
 [![mitmproxy documentation dev](https://shields.mitmproxy.org/badge/docs-dev-brightgreen.svg)](https://docs.mitmproxy.org/main/)
 
 If you have questions on how to use mitmproxy, please
-ask them on StackOverflow!
+use GitHub Discussions!
 
-[![StackOverflow: mitmproxy](https://shields.mitmproxy.org/stackexchange/stackoverflow/t/mitmproxy?color=orange&label=stackoverflow%20questions)](https://stackoverflow.com/questions/tagged/mitmproxy)
+[![mitmproxy discussions](https://shields.mitmproxy.org/badge/help-github%20discussions-orange.svg)](https://github.com/mitmproxy/mitmproxy/discussions)
 
 ## Contributing
 

docs/build.py

@@ -10,7 +10,9 @@
     print(f"Generating output for {script.name}...")
     out = subprocess.check_output(["python3", script.absolute()], cwd=here, text=True)
     if out:
-        (here / "src" / "generated" / f"{script.stem}.html").write_text(out, encoding="utf8")
+        (here / "src" / "generated" / f"{script.stem}.html").write_text(
+            out, encoding="utf8"
+        )
 
 if (here / "public").exists():
     shutil.rmtree(here / "public")