From 98a3e33477c95c7b71e9124d20f26b2ca125c0f7 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 4 Sep 2021 00:24:33 +0200 Subject: [PATCH 001/285] tls: add `tls_handshake`, ignore-after-clienthello this fixes #4702 --- CHANGELOG.md | 8 +- examples/contrib/tls_passthrough.py | 148 +++++++++--------------- mitmproxy/addons/tlsconfig.py | 4 +- mitmproxy/proxy/layers/tls.py | 49 ++++++-- mitmproxy/proxy/server.py | 4 +- test/mitmproxy/addons/test_tlsconfig.py | 14 +-- test/mitmproxy/proxy/layers/test_tls.py | 60 ++++++++-- 7 files changed, 161 insertions(+), 126 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6266a14ce..4a22948810 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,11 +3,13 @@ ## Unreleased: mitmproxy next * Support proxy authentication for SOCKS v5 mode (@starplanet) +* Make it possible to ignore connections in the tls_clienthello event hook (@mhils) +* Add `tls_handshake` event hook to record negotiation success/failure (@mhils) * fix some responses not being decoded properly if the encoding was uppercase #4735 (@Mattwmaster58) -* Expose TLS 1.0 as possible minimum version on older pyOpenSSL releases -* Improve error message on TLS version mismatch. +* Expose TLS 1.0 as possible minimum version on older pyOpenSSL releases (@mhils) +* Improve error message on TLS version mismatch. (@mhils) * Windows: Switch to Python's default asyncio event loop, which increases the number of sockets - that can be processed simultaneously. + that can be processed simultaneously (@mhils) ## 4 August 2021: mitmproxy 7.0.2 diff --git a/examples/contrib/tls_passthrough.py b/examples/contrib/tls_passthrough.py index 8f84f318f9..d248bc36b0 100644 --- a/examples/contrib/tls_passthrough.py +++ b/examples/contrib/tls_passthrough.py @@ -1,8 +1,5 @@ -# FIXME: This addon is currently not compatible with mitmproxy 7 and above. - """ -This inline script allows conditional TLS Interception based -on a user-defined strategy. +This addon allows conditional TLS Interception based on a user-defined strategy. Example: @@ -11,138 +8,103 @@ 1. curl --proxy http://localhost:8080 https://example.com --insecure // works - we'll also see the contents in mitmproxy - 2. curl --proxy http://localhost:8080 https://example.com --insecure - // still works - we'll also see the contents in mitmproxy - - 3. curl --proxy http://localhost:8080 https://example.com + 2. curl --proxy http://localhost:8080 https://example.com // fails with a certificate error, which we will also see in mitmproxy - 4. curl --proxy http://localhost:8080 https://example.com + 3. curl --proxy http://localhost:8080 https://example.com // works again, but mitmproxy does not intercept and we do *not* see the contents - -Authors: Maximilian Hils, Matthew Tuusberg """ import collections import random - +from abc import ABC, abstractmethod from enum import Enum -import mitmproxy -from mitmproxy import ctx -from mitmproxy.exceptions import TlsProtocolException -from mitmproxy.proxy.protocol import TlsLayer, RawTCPLayer +from mitmproxy import connection, ctx +from mitmproxy.proxy.layers import tls +from mitmproxy.utils import human class InterceptionResult(Enum): - success = True - failure = False - skipped = None + SUCCESS = 1 + FAILURE = 2 + SKIPPED = 3 -class _TlsStrategy: - """ - Abstract base class for interception strategies. - """ - +class TlsStrategy(ABC): def __init__(self): # A server_address -> interception results mapping self.history = collections.defaultdict(lambda: collections.deque(maxlen=200)) - def should_intercept(self, server_address): - """ - Returns: - True, if we should attempt to intercept the connection. - False, if we want to employ pass-through instead. - """ + @abstractmethod + def should_intercept(self, server_address: connection.Address) -> bool: raise NotImplementedError() def record_success(self, server_address): - self.history[server_address].append(InterceptionResult.success) + self.history[server_address].append(InterceptionResult.SUCCESS) def record_failure(self, server_address): - self.history[server_address].append(InterceptionResult.failure) + self.history[server_address].append(InterceptionResult.FAILURE) def record_skipped(self, server_address): - self.history[server_address].append(InterceptionResult.skipped) + self.history[server_address].append(InterceptionResult.SKIPPED) -class ConservativeStrategy(_TlsStrategy): +class ConservativeStrategy(TlsStrategy): """ Conservative Interception Strategy - only intercept if there haven't been any failed attempts in the history. """ - - def should_intercept(self, server_address): - if InterceptionResult.failure in self.history[server_address]: - return False - return True + def should_intercept(self, server_address: connection.Address) -> bool: + return InterceptionResult.FAILURE not in self.history[server_address] -class ProbabilisticStrategy(_TlsStrategy): +class ProbabilisticStrategy(TlsStrategy): """ Fixed probability that we intercept a given connection. """ - - def __init__(self, p): + def __init__(self, p: float): self.p = p super().__init__() - def should_intercept(self, server_address): + def should_intercept(self, server_address: connection.Address) -> bool: return random.uniform(0, 1) < self.p -class TlsFeedback(TlsLayer): - """ - Monkey-patch _establish_tls_with_client to get feedback if TLS could be established - successfully on the client connection (which may fail due to cert pinning). - """ +class MaybeTls: + strategy: TlsStrategy - def _establish_tls_with_client(self): - server_address = self.server_conn.address + def load(self, l): + l.add_option( + "tls_strategy", int, 0, + "TLS passthrough strategy. If set to 0, connections will be passed through after the first unsuccessful " + "handshake. If set to 0 < p <= 100, connections with be passed through with probability p.", + ) - try: - super()._establish_tls_with_client() - except TlsProtocolException as e: - tls_strategy.record_failure(server_address) - raise e + def configure(self, updated): + if "tls_strategy" not in updated: + return + if ctx.options.tls_strategy > 0: + self.strategy = ProbabilisticStrategy(ctx.options.tls_strategy / 100) else: - tls_strategy.record_success(server_address) - - -# inline script hooks below. - -tls_strategy = None - - -def load(l): - l.add_option( - "tlsstrat", int, 0, "TLS passthrough strategy (0-100)", - ) - - -def configure(updated): - global tls_strategy - if ctx.options.tlsstrat > 0: - tls_strategy = ProbabilisticStrategy(float(ctx.options.tlsstrat) / 100.0) - else: - tls_strategy = ConservativeStrategy() - + self.strategy = ConservativeStrategy() + + def tls_clienthello(self, data: tls.ClientHelloData): + server_address = data.context.server.peername + if not self.strategy.should_intercept(server_address): + ctx.log(f"TLS passthrough: {human.format_address(server_address)}.") + data.ignore_connection = True + self.strategy.record_skipped(server_address) + + def tls_handshake(self, data: tls.TlsHookData): + if isinstance(data.conn, connection.Server): + return + server_address = data.context.server.peername + if data.conn.error is None: + ctx.log(f"TLS handshake successful: {human.format_address(server_address)}") + self.strategy.record_success(server_address) + else: + ctx.log(f"TLS handshake failed: {human.format_address(server_address)}") + self.strategy.record_failure(server_address) -def next_layer(next_layer): - """ - This hook does the actual magic - if the next layer is planned to be a TLS layer, - we check if we want to enter pass-through mode instead. - """ - if isinstance(next_layer, TlsLayer) and next_layer._client_tls: - server_address = next_layer.server_conn.address - if tls_strategy.should_intercept(server_address): - # We try to intercept. - # Monkey-Patch the layer to get feedback from the TLSLayer if interception worked. - next_layer.__class__ = TlsFeedback - else: - # We don't intercept - reply with a pass-through layer and add a "skipped" entry. - mitmproxy.ctx.log("TLS passthrough for %s" % repr(next_layer.server_conn.address), "info") - next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True) - next_layer.reply.send(next_layer_replacement) - tls_strategy.record_skipped(server_address) +addons = [MaybeTls()] diff --git a/mitmproxy/addons/tlsconfig.py b/mitmproxy/addons/tlsconfig.py index 0c8ebf92a1..9e2a57bd97 100644 --- a/mitmproxy/addons/tlsconfig.py +++ b/mitmproxy/addons/tlsconfig.py @@ -112,7 +112,7 @@ def tls_clienthello(self, tls_clienthello: tls.ClientHelloData): ctx.options.upstream_cert ) - def tls_start_client(self, tls_start: tls.TlsStartData) -> None: + def tls_start_client(self, tls_start: tls.TlsHookData) -> None: """Establish TLS between client and proxy.""" client: connection.Client = tls_start.context.client server: connection.Server = tls_start.context.server @@ -159,7 +159,7 @@ def tls_start_client(self, tls_start: tls.TlsStartData) -> None: )) tls_start.ssl_conn.set_accept_state() - def tls_start_server(self, tls_start: tls.TlsStartData) -> None: + def tls_start_server(self, tls_start: tls.TlsHookData) -> None: """Establish TLS between proxy and server.""" client: connection.Client = tls_start.context.client server: connection.Server = tls_start.context.server diff --git a/mitmproxy/proxy/layers/tls.py b/mitmproxy/proxy/layers/tls.py index bf65292601..86289bfd81 100644 --- a/mitmproxy/proxy/layers/tls.py +++ b/mitmproxy/proxy/layers/tls.py @@ -9,6 +9,7 @@ from mitmproxy.proxy import commands, events, layer, tunnel from mitmproxy.proxy import context from mitmproxy.proxy.commands import StartHook +from mitmproxy.proxy.layers import tcp from mitmproxy.utils import human @@ -103,6 +104,10 @@ class ClientHelloData: """The context object for this connection.""" client_hello: net_tls.ClientHello """The entire parsed TLS ClientHello.""" + ignore_connection: bool = False + """ + If set to `True`, do not intercept this connection and forward encrypted contents unmodified. + """ establish_server_tls_first: bool = False """ If set to `True`, pause this handshake and establish TLS with an upstream server first. @@ -122,7 +127,7 @@ class TlsClienthelloHook(StartHook): @dataclass -class TlsStartData: +class TlsHookData: conn: connection.Connection context: context.Context ssl_conn: Optional[SSL.Connection] = None @@ -131,23 +136,33 @@ class TlsStartData: @dataclass class TlsStartClientHook(StartHook): """ - TLS Negotation between mitmproxy and a client is about to start. + TLS negotation between mitmproxy and a client is about to start. An addon is expected to initialize data.ssl_conn. - (by default, this is done by mitmproxy.addons.TlsConfig) + (by default, this is done by `mitmproxy.addons.tlsconfig`) """ - data: TlsStartData + data: TlsHookData @dataclass class TlsStartServerHook(StartHook): """ - TLS Negotation between mitmproxy and a server is about to start. + TLS negotation between mitmproxy and a server is about to start. An addon is expected to initialize data.ssl_conn. - (by default, this is done by mitmproxy.addons.TlsConfig) + (by default, this is done by `mitmproxy.addons.tlsconfig`) + """ + data: TlsHookData + + +@dataclass +class TlsHandshakeHook(StartHook): """ - data: TlsStartData + A TLS handshake has been completed. + + If `data.conn.error` is `None`, negotiation was successful. + """ + data: TlsHookData class _TLSLayer(tunnel.TunnelLayer): @@ -169,7 +184,7 @@ def __repr__(self): def start_tls(self) -> layer.CommandGenerator[None]: assert not self.tls - tls_start = TlsStartData(self.conn, self.context) + tls_start = TlsHookData(self.conn, self.context) if tls_start.conn == tls_start.context.client: yield TlsStartClientHook(tls_start) else: @@ -220,7 +235,6 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[Tuple[bo ) else: err = f"OpenSSL {e!r}" - self.conn.error = err return False, err else: # Here we set all attributes that are only known *after* the handshake. @@ -242,9 +256,15 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[Tuple[bo self.conn.tls_version = self.tls.get_protocol_version_name() if self.debug: yield commands.Log(f"{self.debug}[tls] tls established: {self.conn}", "debug") + yield TlsHandshakeHook(TlsHookData(self.conn, self.context, self.tls)) yield from self.receive_data(b"") return True, None + def on_handshake_error(self, err: str) -> layer.CommandGenerator[None]: + self.conn.error = err + yield TlsHandshakeHook(TlsHookData(self.conn, self.context, self.tls)) + yield from super().on_handshake_error(err) + def receive_data(self, data: bytes) -> layer.CommandGenerator[None]: if data: self.tls.bio_write(data) @@ -400,6 +420,17 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[Tuple[bo tls_clienthello = ClientHelloData(self.context, client_hello) yield TlsClienthelloHook(tls_clienthello) + if tls_clienthello.ignore_connection: + # we've figured out that we don't want to intercept this connection, so we assign fake connection objects + # to all TLS layers. This makes the real connection contents just go through. + self.conn = self.tunnel_connection = connection.Client(("ignore-conn", 0), ("ignore-conn", 0), time.time()) + parent_layer = self.context.layers[-2] + if isinstance(parent_layer, ServerTLSLayer): + parent_layer.conn = parent_layer.tunnel_connection = connection.Server(None) + self.child_layer = tcp.TCPLayer(self.context, ignore=True) + yield from self.event_to_child(events.DataReceived(self.context.client, bytes(self.recv_buffer))) + self.recv_buffer.clear() + return True, None if tls_clienthello.establish_server_tls_first and not self.context.server.tls_established: err = yield from self.start_server_tls() if err: diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py index cb434f650c..50c1c47213 100644 --- a/mitmproxy/proxy/server.py +++ b/mitmproxy/proxy/server.py @@ -414,7 +414,7 @@ def request(flow: http.HTTPFlow): if "redirect" in flow.request.path: flow.request.host = "httpbin.org" - def tls_start_client(tls_start: tls.TlsStartData): + def tls_start_client(tls_start: tls.TlsHookData): # INSECURE ssl_context = SSL.Context(SSL.SSLv23_METHOD) ssl_context.use_privatekey_file( @@ -426,7 +426,7 @@ def tls_start_client(tls_start: tls.TlsStartData): tls_start.ssl_conn = SSL.Connection(ssl_context) tls_start.ssl_conn.set_accept_state() - def tls_start_server(tls_start: tls.TlsStartData): + def tls_start_server(tls_start: tls.TlsHookData): # INSECURE ssl_context = SSL.Context(SSL.SSLv23_METHOD) tls_start.ssl_conn = SSL.Connection(ssl_context) diff --git a/test/mitmproxy/addons/test_tlsconfig.py b/test/mitmproxy/addons/test_tlsconfig.py index cf407c9ed3..268fd355ff 100644 --- a/test/mitmproxy/addons/test_tlsconfig.py +++ b/test/mitmproxy/addons/test_tlsconfig.py @@ -130,7 +130,7 @@ def test_tls_start_client(self, tdata): ) ctx = context.Context(connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), tctx.options) - tls_start = tls.TlsStartData(ctx.client, context=ctx) + tls_start = tls.TlsHookData(ctx.client, context=ctx) ta.tls_start_client(tls_start) tssl_server = tls_start.ssl_conn tssl_client = test_tls.SSLTest() @@ -145,7 +145,7 @@ def test_tls_start_server_verify_failed(self): ctx.client.cipher_list = ["TLS_AES_256_GCM_SHA384", "ECDHE-RSA-AES128-SHA"] ctx.server.address = ("example.mitmproxy.org", 443) - tls_start = tls.TlsStartData(ctx.server, context=ctx) + tls_start = tls.TlsHookData(ctx.server, context=ctx) ta.tls_start_server(tls_start) tssl_client = tls_start.ssl_conn tssl_server = test_tls.SSLTest(server_side=True) @@ -160,7 +160,7 @@ def test_tls_start_server_verify_ok(self, tdata): tctx.configure(ta, ssl_verify_upstream_trusted_ca=tdata.path( "mitmproxy/net/data/verificationcerts/trusted-root.crt")) - tls_start = tls.TlsStartData(ctx.server, context=ctx) + tls_start = tls.TlsHookData(ctx.server, context=ctx) ta.tls_start_server(tls_start) tssl_client = tls_start.ssl_conn tssl_server = test_tls.SSLTest(server_side=True) @@ -179,7 +179,7 @@ def test_tls_start_server_insecure(self): http2=False, ciphers_server="ALL" ) - tls_start = tls.TlsStartData(ctx.server, context=ctx) + tls_start = tls.TlsHookData(ctx.server, context=ctx) ta.tls_start_server(tls_start) tssl_client = tls_start.ssl_conn tssl_server = test_tls.SSLTest(server_side=True) @@ -190,7 +190,7 @@ def test_alpn_selection(self): with taddons.context(ta) as tctx: ctx = context.Context(connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), tctx.options) ctx.server.address = ("example.mitmproxy.org", 443) - tls_start = tls.TlsStartData(ctx.server, context=ctx) + tls_start = tls.TlsHookData(ctx.server, context=ctx) def assert_alpn(http2, client_offers, expected): tctx.configure(ta, http2=http2) @@ -222,7 +222,7 @@ def test_no_h2_proxy(self, tdata): modes.HttpProxy(ctx), 123 ] - tls_start = tls.TlsStartData(ctx.client, context=ctx) + tls_start = tls.TlsHookData(ctx.client, context=ctx) ta.tls_start_client(tls_start) assert tls_start.ssl_conn.get_app_data()["client_alpn"] == b"http/1.1" @@ -244,7 +244,7 @@ def test_client_cert_file(self, tdata, client_certs): ssl_verify_upstream_trusted_ca=tdata.path("mitmproxy/net/data/verificationcerts/trusted-root.crt"), ) - tls_start = tls.TlsStartData(ctx.server, context=ctx) + tls_start = tls.TlsHookData(ctx.server, context=ctx) ta.tls_start_server(tls_start) tssl_client = tls_start.ssl_conn tssl_server = test_tls.SSLTest(server_side=True) diff --git a/test/mitmproxy/proxy/layers/test_tls.py b/test/mitmproxy/proxy/layers/test_tls.py index 0f6b3da41f..3a8e1a160a 100644 --- a/test/mitmproxy/proxy/layers/test_tls.py +++ b/test/mitmproxy/proxy/layers/test_tls.py @@ -153,13 +153,17 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: yield from super()._handle_event(event) -def interact(playbook: tutils.Playbook, conn: connection.Connection, tssl: SSLTest): +def finish_handshake(playbook: tutils.Playbook, conn: connection.Connection, tssl: SSLTest): data = tutils.Placeholder(bytes) + tls_hook_data = tutils.Placeholder(tls.TlsHookData) assert ( playbook >> events.DataReceived(conn, tssl.bio_read()) + << tls.TlsHandshakeHook(tls_hook_data) + >> tutils.reply() << commands.SendData(conn, data) ) + assert tls_hook_data().conn.error is None tssl.bio_write(data()) @@ -168,7 +172,7 @@ def reply_tls_start_client(alpn: typing.Optional[bytes] = None, *args, **kwargs) Helper function to simplify the syntax for tls_start_client hooks. """ - def make_client_conn(tls_start: tls.TlsStartData) -> None: + def make_client_conn(tls_start: tls.TlsHookData) -> None: # ssl_context = SSL.Context(Method.TLS_METHOD) # ssl_context.set_min_proto_version(SSL.TLS1_3_VERSION) ssl_context = SSL.Context(SSL.SSLv23_METHOD) @@ -193,7 +197,7 @@ def reply_tls_start_server(alpn: typing.Optional[bytes] = None, *args, **kwargs) Helper function to simplify the syntax for tls_start_server hooks. """ - def make_server_conn(tls_start: tls.TlsStartData) -> None: + def make_server_conn(tls_start: tls.TlsHookData) -> None: # ssl_context = SSL.Context(Method.TLS_METHOD) # ssl_context.set_min_proto_version(SSL.TLS1_3_VERSION) ssl_context = SSL.Context(SSL.SSLv23_METHOD) @@ -251,7 +255,7 @@ def test_simple(self, tctx): tssl = SSLTest(server_side=True) - # send ClientHello + # send ClientHello, receive ClientHello data = tutils.Placeholder(bytes) assert ( playbook @@ -259,14 +263,14 @@ def test_simple(self, tctx): >> reply_tls_start_server() << commands.SendData(tctx.server, data) ) - - # receive ServerHello, finish client handshake tssl.bio_write(data()) with pytest.raises(ssl.SSLWantReadError): tssl.do_handshake() - interact(playbook, tctx.server, tssl) - # finish server handshake + # finish handshake (mitmproxy) + finish_handshake(playbook, tctx.server, tssl) + + # finish handshake (locally) tssl.do_handshake() assert ( playbook @@ -323,14 +327,18 @@ def test_untrusted_cert(self, tctx): with pytest.raises(ssl.SSLWantReadError): tssl.do_handshake() + tls_hook_data = tutils.Placeholder(tls.TlsHookData) assert ( playbook >> events.DataReceived(tctx.server, tssl.bio_read()) << commands.Log("Server TLS handshake failed. Certificate verify failed: Hostname mismatch", "warn") + << tls.TlsHandshakeHook(tls_hook_data) + >> tutils.reply() << commands.CloseConnection(tctx.server) << commands.SendData(tctx.client, b"open-connection failed: Certificate verify failed: Hostname mismatch") ) + assert tls_hook_data().conn.error == "Certificate verify failed: Hostname mismatch" assert not tctx.server.tls_established def test_remote_speaks_no_tls(self, tctx): @@ -340,6 +348,7 @@ def test_remote_speaks_no_tls(self, tctx): # send ClientHello, receive random garbage back data = tutils.Placeholder(bytes) + tls_hook_data = tutils.Placeholder(tls.TlsHookData) assert ( playbook << tls.TlsStartServerHook(tutils.Placeholder()) @@ -347,8 +356,11 @@ def test_remote_speaks_no_tls(self, tctx): << commands.SendData(tctx.server, data) >> events.DataReceived(tctx.server, b"HTTP/1.1 404 Not Found\r\n") << commands.Log("Server TLS handshake failed. The remote server does not speak TLS.", "warn") + << tls.TlsHandshakeHook(tls_hook_data) + >> tutils.reply() << commands.CloseConnection(tctx.server) ) + assert tls_hook_data().conn.error == "The remote server does not speak TLS." def test_unsupported_protocol(self, tctx: context.Context): """Test the scenario where the server only supports an outdated TLS version by default.""" @@ -375,13 +387,17 @@ def test_unsupported_protocol(self, tctx: context.Context): tssl.do_handshake() # send back error + tls_hook_data = tutils.Placeholder(tls.TlsHookData) assert ( playbook >> events.DataReceived(tctx.server, tssl.bio_read()) << commands.Log("Server TLS handshake failed. The remote server and mitmproxy cannot agree on a TLS version" " to use. You may need to adjust mitmproxy's tls_version_server_min option.", "warn") + << tls.TlsHandshakeHook(tls_hook_data) + >> tutils.reply() << commands.CloseConnection(tctx.server) ) + assert tls_hook_data().conn.error def make_client_tls_layer( @@ -429,7 +445,7 @@ def test_client_only(self, tctx: context.Context): tssl_client.bio_write(data()) tssl_client.do_handshake() # Finish Handshake - interact(playbook, tctx.client, tssl_client) + finish_handshake(playbook, tctx.client, tssl_client) assert tssl_client.obj.getpeercert(True) assert tctx.client.tls_established @@ -488,6 +504,8 @@ def require_server_conn(client_hello: tls.ClientHelloData) -> None: assert ( playbook >> events.DataReceived(tctx.server, tssl_server.bio_read()) + << tls.TlsHandshakeHook(tutils.Placeholder()) + >> tutils.reply() << commands.SendData(tctx.server, data) << tls.TlsStartClientHook(tutils.Placeholder()) ) @@ -503,7 +521,7 @@ def require_server_conn(client_hello: tls.ClientHelloData) -> None: ) tssl_client.bio_write(data()) tssl_client.do_handshake() - interact(playbook, tctx.client, tssl_client) + finish_handshake(playbook, tctx.client, tssl_client) # Both handshakes completed! assert tctx.client.tls_established @@ -517,6 +535,7 @@ def require_server_conn(client_hello: tls.ClientHelloData) -> None: def test_cannot_parse_clienthello(self, tctx: context.Context): """Test the scenario where we cannot parse the ClientHello""" playbook, client_layer, tssl_client = make_client_tls_layer(tctx) + tls_hook_data = tutils.Placeholder(tls.TlsHookData) invalid = b"\x16\x03\x01\x00\x00" @@ -524,8 +543,11 @@ def test_cannot_parse_clienthello(self, tctx: context.Context): playbook >> events.DataReceived(tctx.client, invalid) << commands.Log(f"Client TLS handshake failed. Cannot parse ClientHello: {invalid.hex()}", level="warn") + << tls.TlsHandshakeHook(tls_hook_data) + >> tutils.reply() << commands.CloseConnection(tctx.client) ) + assert tls_hook_data().conn.error assert not tctx.client.tls_established # Make sure that an active server connection does not cause child layers to spawn. @@ -556,15 +578,19 @@ def test_mitmproxy_ca_is_untrusted(self, tctx: context.Context): with pytest.raises(ssl.SSLCertVerificationError): tssl_client.do_handshake() # Finish Handshake + tls_hook_data = tutils.Placeholder(tls.TlsHookData) assert ( playbook >> events.DataReceived(tctx.client, tssl_client.bio_read()) << commands.Log("Client TLS handshake failed. The client does not trust the proxy's certificate " "for wrong.host.mitmproxy.org (sslv3 alert bad certificate)", "warn") + << tls.TlsHandshakeHook(tls_hook_data) + >> tutils.reply() << commands.CloseConnection(tctx.client) >> events.ConnectionClosed(tctx.client) ) assert not tctx.client.tls_established + assert tls_hook_data().conn.error @pytest.mark.parametrize("close_at", ["tls_clienthello", "tls_start_client", "handshake"]) def test_immediate_disconnect(self, tctx: context.Context, close_at): @@ -573,6 +599,7 @@ def test_immediate_disconnect(self, tctx: context.Context, close_at): the proxy certificate.""" playbook, client_layer, tssl_client = make_client_tls_layer(tctx, sni=b"wrong.host.mitmproxy.org") playbook.logs = True + tls_hook_data = tutils.Placeholder(tls.TlsHookData) playbook >> events.DataReceived(tctx.client, tssl_client.bio_read()) playbook << tls.TlsClienthelloHook(tutils.Placeholder()) @@ -584,8 +611,11 @@ def test_immediate_disconnect(self, tctx: context.Context, close_at): >> tutils.reply(to=-2) << tls.TlsStartClientHook(tutils.Placeholder()) >> reply_tls_start_client() + << tls.TlsHandshakeHook(tls_hook_data) + >> tutils.reply() << commands.CloseConnection(tctx.client) ) + assert tls_hook_data().conn.error return playbook >> tutils.reply() @@ -596,8 +626,11 @@ def test_immediate_disconnect(self, tctx: context.Context, close_at): playbook >> events.ConnectionClosed(tctx.client) >> reply_tls_start_client(to=-2) + << tls.TlsHandshakeHook(tls_hook_data) + >> tutils.reply() << commands.CloseConnection(tctx.client) ) + assert tls_hook_data().conn.error return assert ( @@ -608,14 +641,18 @@ def test_immediate_disconnect(self, tctx: context.Context, close_at): << commands.Log("Client TLS handshake failed. The client disconnected during the handshake. " "If this happens consistently for wrong.host.mitmproxy.org, this may indicate that the " "client does not trust the proxy's certificate.", "info") + << tls.TlsHandshakeHook(tls_hook_data) + >> tutils.reply() << commands.CloseConnection(tctx.client) ) + assert tls_hook_data().conn.error def test_unsupported_protocol(self, tctx: context.Context): """Test the scenario where the client only supports an outdated TLS version by default.""" playbook, client_layer, tssl_client = make_client_tls_layer(tctx, max_ver=ssl.TLSVersion.TLSv1_2) playbook.logs = True + tls_hook_data = tutils.Placeholder(tls.TlsHookData) assert ( playbook >> events.DataReceived(tctx.client, tssl_client.bio_read()) @@ -625,5 +662,8 @@ def test_unsupported_protocol(self, tctx: context.Context): >> reply_tls_start_client() << commands.Log("Client TLS handshake failed. Client and mitmproxy cannot agree on a TLS version to " "use. You may need to adjust mitmproxy's tls_version_client_min option.", "warn") + << tls.TlsHandshakeHook(tls_hook_data) + >> tutils.reply() << commands.CloseConnection(tctx.client) ) + assert tls_hook_data().conn.error From 9f39e2f387463f3cfcd044b7c7135865e06c506d Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 4 Sep 2021 16:03:06 +0200 Subject: [PATCH 002/285] tests++ --- docs/scripts/api-events.py | 1 + examples/contrib/tls_passthrough.py | 2 +- mitmproxy/proxy/layers/tls.py | 2 +- test/mitmproxy/proxy/layers/test_tls.py | 43 ++++++++++++++++++++++--- 4 files changed, 42 insertions(+), 6 deletions(-) diff --git a/docs/scripts/api-events.py b/docs/scripts/api-events.py index 80d91dae9b..462c025310 100644 --- a/docs/scripts/api-events.py +++ b/docs/scripts/api-events.py @@ -124,6 +124,7 @@ def category(name: str, desc: str, hooks: List[Type[hooks.Hook]]) -> None: tls.TlsClienthelloHook, tls.TlsStartClientHook, tls.TlsStartServerHook, + tls.TlsHandshakeHook, ] ) diff --git a/examples/contrib/tls_passthrough.py b/examples/contrib/tls_passthrough.py index d248bc36b0..8652651f28 100644 --- a/examples/contrib/tls_passthrough.py +++ b/examples/contrib/tls_passthrough.py @@ -97,7 +97,7 @@ def tls_clienthello(self, data: tls.ClientHelloData): def tls_handshake(self, data: tls.TlsHookData): if isinstance(data.conn, connection.Server): - return + return # we are only interested in failing client connections here. server_address = data.context.server.peername if data.conn.error is None: ctx.log(f"TLS handshake successful: {human.format_address(server_address)}") diff --git a/mitmproxy/proxy/layers/tls.py b/mitmproxy/proxy/layers/tls.py index 86289bfd81..34c7cea7ec 100644 --- a/mitmproxy/proxy/layers/tls.py +++ b/mitmproxy/proxy/layers/tls.py @@ -424,7 +424,7 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[Tuple[bo # we've figured out that we don't want to intercept this connection, so we assign fake connection objects # to all TLS layers. This makes the real connection contents just go through. self.conn = self.tunnel_connection = connection.Client(("ignore-conn", 0), ("ignore-conn", 0), time.time()) - parent_layer = self.context.layers[-2] + parent_layer = self.context.layers[self.context.layers.index(self) - 1] if isinstance(parent_layer, ServerTLSLayer): parent_layer.conn = parent_layer.tunnel_connection = connection.Server(None) self.child_layer = tcp.TCPLayer(self.context, ignore=True) diff --git a/test/mitmproxy/proxy/layers/test_tls.py b/test/mitmproxy/proxy/layers/test_tls.py index 3a8e1a160a..4c4aad7d15 100644 --- a/test/mitmproxy/proxy/layers/test_tls.py +++ b/test/mitmproxy/proxy/layers/test_tls.py @@ -1,4 +1,5 @@ import ssl +import time import typing import pytest @@ -459,13 +460,13 @@ def test_client_only(self, tctx: context.Context): << commands.SendData(other_server, b"plaintext") ) - @pytest.mark.parametrize("eager", ["eager", ""]) - def test_server_required(self, tctx, eager): + @pytest.mark.parametrize("server_state", ["open", "closed"]) + def test_server_required(self, tctx, server_state): """ Test the scenario where a server connection is required (for example, because of an unknown ALPN) to establish TLS with the client. """ - if eager: + if server_state == "open": tctx.server.state = ConnectionState.OPEN tssl_server = SSLTest(server_side=True, alpn=["quux"]) playbook, client_layer, tssl_client = make_client_tls_layer(tctx, alpn=["quux"]) @@ -482,7 +483,7 @@ def require_server_conn(client_hello: tls.ClientHelloData) -> None: << tls.TlsClienthelloHook(tutils.Placeholder()) >> tutils.reply(side_effect=require_server_conn) ) - if not eager: + if server_state == "closed": ( playbook << commands.OpenConnection(tctx.server) @@ -532,6 +533,40 @@ def require_server_conn(client_hello: tls.ClientHelloData) -> None: _test_echo(playbook, tssl_server, tctx.server) _test_echo(playbook, tssl_client, tctx.client) + @pytest.mark.parametrize("server_state", ["open", "closed"]) + def test_passthrough_from_clienthello(self, tctx, server_state): + """ + Test the scenario where the connection is moved to passthrough mode in the tls_clienthello hook. + """ + if server_state == "open": + tctx.server.timestamp_start = time.time() + tctx.server.state = ConnectionState.OPEN + + playbook, client_layer, tssl_client = make_client_tls_layer(tctx, alpn=["quux"]) + + def make_passthrough(client_hello: tls.ClientHelloData) -> None: + client_hello.ignore_connection = True + + client_hello = tssl_client.bio_read() + ( + playbook + >> events.DataReceived(tctx.client, client_hello) + << tls.TlsClienthelloHook(tutils.Placeholder()) + >> tutils.reply(side_effect=make_passthrough) + ) + if server_state == "closed": + ( + playbook + << commands.OpenConnection(tctx.server) + >> tutils.reply(None) + ) + assert ( + playbook + << commands.SendData(tctx.server, client_hello) # passed through unmodified + >> events.DataReceived(tctx.server, b"ServerHello") # and the same for the serverhello. + << commands.SendData(tctx.client, b"ServerHello") + ) + def test_cannot_parse_clienthello(self, tctx: context.Context): """Test the scenario where we cannot parse the ClientHello""" playbook, client_layer, tssl_client = make_client_tls_layer(tctx) From bdf4e31c588046041b0f5b019ad19aaf9e06ffe4 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 4 Sep 2021 16:10:39 +0200 Subject: [PATCH 003/285] move `ClientHello` to `mitmproxy.tls` --- mitmproxy/net/tls.py | 50 ------------------ mitmproxy/proxy/layers/tls.py | 8 +-- mitmproxy/tls.py | 53 ++++++++++++++++++++ test/mitmproxy/net/test_tls.py | 51 ------------------- test/mitmproxy/proxy/layers/test_tls_fuzz.py | 4 +- test/mitmproxy/test_tls.py | 53 ++++++++++++++++++++ 6 files changed, 112 insertions(+), 107 deletions(-) create mode 100644 mitmproxy/tls.py create mode 100644 test/mitmproxy/test_tls.py diff --git a/mitmproxy/net/tls.py b/mitmproxy/net/tls.py index 7e85fd55e9..8ac938aeb9 100644 --- a/mitmproxy/net/tls.py +++ b/mitmproxy/net/tls.py @@ -1,4 +1,3 @@ -import io import ipaddress import os import threading @@ -11,12 +10,9 @@ from OpenSSL.crypto import X509 from cryptography.hazmat.primitives.asymmetric import rsa -from kaitaistruct import KaitaiStream from OpenSSL import SSL, crypto from mitmproxy import certs -from mitmproxy.contrib.kaitaistruct import tls_client_hello -from mitmproxy.net import check # redeclared here for strict type checking @@ -279,49 +275,3 @@ def is_tls_record_magic(d): d[1] == 0x03 and 0x0 <= d[2] <= 0x03 ) - - -class ClientHello: - - def __init__(self, raw_client_hello): - self._client_hello = tls_client_hello.TlsClientHello( - KaitaiStream(io.BytesIO(raw_client_hello)) - ) - - @property - def cipher_suites(self) -> List[int]: - return self._client_hello.cipher_suites.cipher_suites - - @property - def sni(self) -> Optional[str]: - if self._client_hello.extensions: - for extension in self._client_hello.extensions.extensions: - is_valid_sni_extension = ( - extension.type == 0x00 and - len(extension.body.server_names) == 1 and - extension.body.server_names[0].name_type == 0 and - check.is_valid_host(extension.body.server_names[0].host_name) - ) - if is_valid_sni_extension: - return extension.body.server_names[0].host_name.decode("ascii") - return None - - @property - def alpn_protocols(self) -> List[bytes]: - if self._client_hello.extensions: - for extension in self._client_hello.extensions.extensions: - if extension.type == 0x10: - return list(x.name for x in extension.body.alpn_protocols) - return [] - - @property - def extensions(self) -> List[Tuple[int, bytes]]: - ret = [] - if self._client_hello.extensions: - for extension in self._client_hello.extensions.extensions: - body = getattr(extension, "_raw_body", extension.body) - ret.append((extension.type, body)) - return ret - - def __repr__(self): - return f"ClientHello(sni: {self.sni}, alpn_protocols: {self.alpn_protocols})" diff --git a/mitmproxy/proxy/layers/tls.py b/mitmproxy/proxy/layers/tls.py index 34c7cea7ec..8c9c4e0bce 100644 --- a/mitmproxy/proxy/layers/tls.py +++ b/mitmproxy/proxy/layers/tls.py @@ -4,8 +4,8 @@ from typing import Iterator, Literal, Optional, Tuple from OpenSSL import SSL +from mitmproxy.tls import ClientHello from mitmproxy import certs, connection -from mitmproxy.net import tls as net_tls from mitmproxy.proxy import commands, events, layer, tunnel from mitmproxy.proxy import context from mitmproxy.proxy.commands import StartHook @@ -70,7 +70,7 @@ def get_client_hello(data: bytes) -> Optional[bytes]: return None -def parse_client_hello(data: bytes) -> Optional[net_tls.ClientHello]: +def parse_client_hello(data: bytes) -> Optional[ClientHello]: """ Check if the supplied bytes contain a full ClientHello message, and if so, parse it. @@ -86,7 +86,7 @@ def parse_client_hello(data: bytes) -> Optional[net_tls.ClientHello]: client_hello = get_client_hello(data) if client_hello: try: - return net_tls.ClientHello(client_hello[4:]) + return ClientHello(client_hello[4:]) except EOFError as e: raise ValueError("Invalid ClientHello") from e return None @@ -102,7 +102,7 @@ def parse_client_hello(data: bytes) -> Optional[net_tls.ClientHello]: class ClientHelloData: context: context.Context """The context object for this connection.""" - client_hello: net_tls.ClientHello + client_hello: ClientHello """The entire parsed TLS ClientHello.""" ignore_connection: bool = False """ diff --git a/mitmproxy/tls.py b/mitmproxy/tls.py new file mode 100644 index 0000000000..4e95f231dd --- /dev/null +++ b/mitmproxy/tls.py @@ -0,0 +1,53 @@ +import io +from typing import List, Optional, Tuple + +from kaitaistruct import KaitaiStream + +from mitmproxy.contrib.kaitaistruct import tls_client_hello +from mitmproxy.net import check + + +class ClientHello: + + def __init__(self, raw_client_hello): + self._client_hello = tls_client_hello.TlsClientHello( + KaitaiStream(io.BytesIO(raw_client_hello)) + ) + + @property + def cipher_suites(self) -> List[int]: + return self._client_hello.cipher_suites.cipher_suites + + @property + def sni(self) -> Optional[str]: + if self._client_hello.extensions: + for extension in self._client_hello.extensions.extensions: + is_valid_sni_extension = ( + extension.type == 0x00 and + len(extension.body.server_names) == 1 and + extension.body.server_names[0].name_type == 0 and + check.is_valid_host(extension.body.server_names[0].host_name) + ) + if is_valid_sni_extension: + return extension.body.server_names[0].host_name.decode("ascii") + return None + + @property + def alpn_protocols(self) -> List[bytes]: + if self._client_hello.extensions: + for extension in self._client_hello.extensions.extensions: + if extension.type == 0x10: + return list(x.name for x in extension.body.alpn_protocols) + return [] + + @property + def extensions(self) -> List[Tuple[int, bytes]]: + ret = [] + if self._client_hello.extensions: + for extension in self._client_hello.extensions.extensions: + body = getattr(extension, "_raw_body", extension.body) + ret.append((extension.type, body)) + return ret + + def __repr__(self): + return f"ClientHello(sni: {self.sni}, alpn_protocols: {self.alpn_protocols})" diff --git a/test/mitmproxy/net/test_tls.py b/test/mitmproxy/net/test_tls.py index 9fe6f0f826..b082a7de65 100644 --- a/test/mitmproxy/net/test_tls.py +++ b/test/mitmproxy/net/test_tls.py @@ -4,17 +4,6 @@ from mitmproxy import certs from mitmproxy.net import tls -CLIENT_HELLO_NO_EXTENSIONS = bytes.fromhex( - "03015658a756ab2c2bff55f636814deac086b7ca56b65058c7893ffc6074f5245f70205658a75475103a152637" - "78e1bb6d22e8bbd5b6b0a3a59760ad354e91ba20d353001a0035002f000a000500040009000300060008006000" - "61006200640100" -) -FULL_CLIENT_HELLO_NO_EXTENSIONS = ( - b"\x16\x03\x03\x00\x65" # record layer - b"\x01\x00\x00\x61" + # handshake header - CLIENT_HELLO_NO_EXTENSIONS -) - def test_make_master_secret_logger(): assert tls.make_master_secret_logger(None) is None @@ -84,43 +73,3 @@ def test_is_record_magic(): assert tls.is_tls_record_magic(b"\x16\x03\x01") assert tls.is_tls_record_magic(b"\x16\x03\x02") assert tls.is_tls_record_magic(b"\x16\x03\x03") - - -class TestClientHello: - def test_no_extensions(self): - c = tls.ClientHello(CLIENT_HELLO_NO_EXTENSIONS) - assert repr(c) - assert c.sni is None - assert c.cipher_suites == [53, 47, 10, 5, 4, 9, 3, 6, 8, 96, 97, 98, 100] - assert c.alpn_protocols == [] - assert c.extensions == [] - - def test_extensions(self): - data = bytes.fromhex( - "03033b70638d2523e1cba15f8364868295305e9c52aceabda4b5147210abc783e6e1000022c02bc02fc02cc030" - "cca9cca8cc14cc13c009c013c00ac014009c009d002f0035000a0100006cff0100010000000010000e00000b65" - "78616d706c652e636f6d0017000000230000000d00120010060106030501050304010403020102030005000501" - "00000000001200000010000e000c02683208687474702f312e3175500000000b00020100000a00080006001d00" - "170018" - ) - c = tls.ClientHello(data) - assert repr(c) - assert c.sni == 'example.com' - assert c.cipher_suites == [ - 49195, 49199, 49196, 49200, 52393, 52392, 52244, 52243, 49161, - 49171, 49162, 49172, 156, 157, 47, 53, 10 - ] - assert c.alpn_protocols == [b'h2', b'http/1.1'] - assert c.extensions == [ - (65281, b'\x00'), - (0, b'\x00\x0e\x00\x00\x0bexample.com'), - (23, b''), - (35, b''), - (13, b'\x00\x10\x06\x01\x06\x03\x05\x01\x05\x03\x04\x01\x04\x03\x02\x01\x02\x03'), - (5, b'\x01\x00\x00\x00\x00'), - (18, b''), - (16, b'\x00\x0c\x02h2\x08http/1.1'), - (30032, b''), - (11, b'\x01\x00'), - (10, b'\x00\x06\x00\x1d\x00\x17\x00\x18') - ] diff --git a/test/mitmproxy/proxy/layers/test_tls_fuzz.py b/test/mitmproxy/proxy/layers/test_tls_fuzz.py index f95b40bb0f..524e691a02 100644 --- a/test/mitmproxy/proxy/layers/test_tls_fuzz.py +++ b/test/mitmproxy/proxy/layers/test_tls_fuzz.py @@ -1,7 +1,7 @@ from hypothesis import given, example from hypothesis.strategies import binary, integers -from mitmproxy.net.tls import ClientHello +from mitmproxy.tls import ClientHello from mitmproxy.proxy.layers.tls import parse_client_hello client_hello_with_extensions = bytes.fromhex( @@ -17,7 +17,7 @@ @given(i=integers(0, len(client_hello_with_extensions)), data=binary()) @example(i=183, data=b'\x00\x00\x00\x00\x00\x00\x00\x00\x00') -def test_fuzz_h2_request_chunks(i, data): +def test_fuzz_parse_client_hello(i, data): try: ch = parse_client_hello(client_hello_with_extensions[:i] + data) except ValueError: diff --git a/test/mitmproxy/test_tls.py b/test/mitmproxy/test_tls.py new file mode 100644 index 0000000000..c8ebc36cc2 --- /dev/null +++ b/test/mitmproxy/test_tls.py @@ -0,0 +1,53 @@ +from mitmproxy import tls + + +CLIENT_HELLO_NO_EXTENSIONS = bytes.fromhex( + "03015658a756ab2c2bff55f636814deac086b7ca56b65058c7893ffc6074f5245f70205658a75475103a152637" + "78e1bb6d22e8bbd5b6b0a3a59760ad354e91ba20d353001a0035002f000a000500040009000300060008006000" + "61006200640100" +) +FULL_CLIENT_HELLO_NO_EXTENSIONS = ( + b"\x16\x03\x03\x00\x65" # record layer + b"\x01\x00\x00\x61" + # handshake header + CLIENT_HELLO_NO_EXTENSIONS +) + + +class TestClientHello: + def test_no_extensions(self): + c = tls.ClientHello(CLIENT_HELLO_NO_EXTENSIONS) + assert repr(c) + assert c.sni is None + assert c.cipher_suites == [53, 47, 10, 5, 4, 9, 3, 6, 8, 96, 97, 98, 100] + assert c.alpn_protocols == [] + assert c.extensions == [] + + def test_extensions(self): + data = bytes.fromhex( + "03033b70638d2523e1cba15f8364868295305e9c52aceabda4b5147210abc783e6e1000022c02bc02fc02cc030" + "cca9cca8cc14cc13c009c013c00ac014009c009d002f0035000a0100006cff0100010000000010000e00000b65" + "78616d706c652e636f6d0017000000230000000d00120010060106030501050304010403020102030005000501" + "00000000001200000010000e000c02683208687474702f312e3175500000000b00020100000a00080006001d00" + "170018" + ) + c = tls.ClientHello(data) + assert repr(c) + assert c.sni == 'example.com' + assert c.cipher_suites == [ + 49195, 49199, 49196, 49200, 52393, 52392, 52244, 52243, 49161, + 49171, 49162, 49172, 156, 157, 47, 53, 10 + ] + assert c.alpn_protocols == [b'h2', b'http/1.1'] + assert c.extensions == [ + (65281, b'\x00'), + (0, b'\x00\x0e\x00\x00\x0bexample.com'), + (23, b''), + (35, b''), + (13, b'\x00\x10\x06\x01\x06\x03\x05\x01\x05\x03\x04\x01\x04\x03\x02\x01\x02\x03'), + (5, b'\x01\x00\x00\x00\x00'), + (18, b''), + (16, b'\x00\x0c\x02h2\x08http/1.1'), + (30032, b''), + (11, b'\x01\x00'), + (10, b'\x00\x06\x00\x1d\x00\x17\x00\x18') + ] From 7fd887a553a87b5d6c316a18f277536cdb1bb5da Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 4 Sep 2021 16:37:39 +0200 Subject: [PATCH 004/285] move tls hook data to `mitmproxy.tls` --- examples/contrib/tls_passthrough.py | 5 +- mitmproxy/addons/tlsconfig.py | 11 +- mitmproxy/proxy/layers/tls.py | 37 +--- mitmproxy/proxy/server.py | 7 +- mitmproxy/tls.py | 51 +++++- test/mitmproxy/addons/test_tlsconfig.py | 22 +-- test/mitmproxy/proxy/layers/test_tls.py | 229 ++++++++++++------------ 7 files changed, 191 insertions(+), 171 deletions(-) diff --git a/examples/contrib/tls_passthrough.py b/examples/contrib/tls_passthrough.py index 8652651f28..182f26289d 100644 --- a/examples/contrib/tls_passthrough.py +++ b/examples/contrib/tls_passthrough.py @@ -19,8 +19,7 @@ from abc import ABC, abstractmethod from enum import Enum -from mitmproxy import connection, ctx -from mitmproxy.proxy.layers import tls +from mitmproxy import connection, ctx, tls from mitmproxy.utils import human @@ -95,7 +94,7 @@ def tls_clienthello(self, data: tls.ClientHelloData): data.ignore_connection = True self.strategy.record_skipped(server_address) - def tls_handshake(self, data: tls.TlsHookData): + def tls_handshake(self, data: tls.TlsData): if isinstance(data.conn, connection.Server): return # we are only interested in failing client connections here. server_address = data.context.server.peername diff --git a/mitmproxy/addons/tlsconfig.py b/mitmproxy/addons/tlsconfig.py index 9e2a57bd97..548a95a879 100644 --- a/mitmproxy/addons/tlsconfig.py +++ b/mitmproxy/addons/tlsconfig.py @@ -4,11 +4,12 @@ from typing import List, Optional, TypedDict, Any from OpenSSL import SSL -from mitmproxy import certs, ctx, exceptions, connection +from mitmproxy import certs, ctx, exceptions, connection, tls from mitmproxy.net import tls as net_tls from mitmproxy.options import CONF_BASENAME from mitmproxy.proxy import context -from mitmproxy.proxy.layers import tls, modes +from mitmproxy.proxy.layers import modes +from mitmproxy.proxy.layers import tls as proxy_tls # We manually need to specify this, otherwise OpenSSL may select a non-HTTP2 cipher by default. # https://ssl-config.mozilla.org/#config=old @@ -46,7 +47,7 @@ def alpn_select_callback(conn: SSL.Connection, options: List[bytes]) -> Any: # We do have a server connection, but the remote server refused to negotiate a protocol: # We need to mirror this on the client connection. return SSL.NO_OVERLAPPING_PROTOCOLS - http_alpns = tls.HTTP_ALPNS if http2 else tls.HTTP1_ALPNS + http_alpns = proxy_tls.HTTP_ALPNS if http2 else proxy_tls.HTTP1_ALPNS for alpn in options: # client sends in order of preference, so we are nice and respect that. if alpn in http_alpns: return alpn @@ -112,7 +113,7 @@ def tls_clienthello(self, tls_clienthello: tls.ClientHelloData): ctx.options.upstream_cert ) - def tls_start_client(self, tls_start: tls.TlsHookData) -> None: + def tls_start_client(self, tls_start: tls.TlsData) -> None: """Establish TLS between client and proxy.""" client: connection.Client = tls_start.context.client server: connection.Server = tls_start.context.server @@ -159,7 +160,7 @@ def tls_start_client(self, tls_start: tls.TlsHookData) -> None: )) tls_start.ssl_conn.set_accept_state() - def tls_start_server(self, tls_start: tls.TlsHookData) -> None: + def tls_start_server(self, tls_start: tls.TlsData) -> None: """Establish TLS between proxy and server.""" client: connection.Client = tls_start.context.client server: connection.Server = tls_start.context.server diff --git a/mitmproxy/proxy/layers/tls.py b/mitmproxy/proxy/layers/tls.py index 8c9c4e0bce..f9896f0df1 100644 --- a/mitmproxy/proxy/layers/tls.py +++ b/mitmproxy/proxy/layers/tls.py @@ -4,7 +4,7 @@ from typing import Iterator, Literal, Optional, Tuple from OpenSSL import SSL -from mitmproxy.tls import ClientHello +from mitmproxy.tls import ClientHello, ClientHelloData, TlsData from mitmproxy import certs, connection from mitmproxy.proxy import commands, events, layer, tunnel from mitmproxy.proxy import context @@ -98,22 +98,6 @@ def parse_client_hello(data: bytes) -> Optional[ClientHello]: # We need these classes as hooks can only have one argument at the moment. -@dataclass -class ClientHelloData: - context: context.Context - """The context object for this connection.""" - client_hello: ClientHello - """The entire parsed TLS ClientHello.""" - ignore_connection: bool = False - """ - If set to `True`, do not intercept this connection and forward encrypted contents unmodified. - """ - establish_server_tls_first: bool = False - """ - If set to `True`, pause this handshake and establish TLS with an upstream server first. - This makes it possible to process the server certificate when generating an interception certificate. - """ - @dataclass class TlsClienthelloHook(StartHook): @@ -126,13 +110,6 @@ class TlsClienthelloHook(StartHook): data: ClientHelloData -@dataclass -class TlsHookData: - conn: connection.Connection - context: context.Context - ssl_conn: Optional[SSL.Connection] = None - - @dataclass class TlsStartClientHook(StartHook): """ @@ -141,7 +118,7 @@ class TlsStartClientHook(StartHook): An addon is expected to initialize data.ssl_conn. (by default, this is done by `mitmproxy.addons.tlsconfig`) """ - data: TlsHookData + data: TlsData @dataclass @@ -152,7 +129,7 @@ class TlsStartServerHook(StartHook): An addon is expected to initialize data.ssl_conn. (by default, this is done by `mitmproxy.addons.tlsconfig`) """ - data: TlsHookData + data: TlsData @dataclass @@ -162,7 +139,7 @@ class TlsHandshakeHook(StartHook): If `data.conn.error` is `None`, negotiation was successful. """ - data: TlsHookData + data: TlsData class _TLSLayer(tunnel.TunnelLayer): @@ -184,7 +161,7 @@ def __repr__(self): def start_tls(self) -> layer.CommandGenerator[None]: assert not self.tls - tls_start = TlsHookData(self.conn, self.context) + tls_start = TlsData(self.conn, self.context) if tls_start.conn == tls_start.context.client: yield TlsStartClientHook(tls_start) else: @@ -256,13 +233,13 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[Tuple[bo self.conn.tls_version = self.tls.get_protocol_version_name() if self.debug: yield commands.Log(f"{self.debug}[tls] tls established: {self.conn}", "debug") - yield TlsHandshakeHook(TlsHookData(self.conn, self.context, self.tls)) + yield TlsHandshakeHook(TlsData(self.conn, self.context, self.tls)) yield from self.receive_data(b"") return True, None def on_handshake_error(self, err: str) -> layer.CommandGenerator[None]: self.conn.error = err - yield TlsHandshakeHook(TlsHookData(self.conn, self.context, self.tls)) + yield TlsHandshakeHook(TlsData(self.conn, self.context, self.tls)) yield from super().on_handshake_error(err) def receive_data(self, data: bytes) -> layer.CommandGenerator[None]: diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py index 50c1c47213..4b26b23c6b 100644 --- a/mitmproxy/proxy/server.py +++ b/mitmproxy/proxy/server.py @@ -16,12 +16,11 @@ from dataclasses import dataclass from OpenSSL import SSL -from mitmproxy import http, options as moptions +from mitmproxy import http, options as moptions, tls from mitmproxy.proxy.context import Context from mitmproxy.proxy.layers.http import HTTPMode from mitmproxy.proxy import commands, events, layer, layers, server_hooks from mitmproxy.connection import Address, Client, Connection, ConnectionState -from mitmproxy.proxy.layers import tls from mitmproxy.utils import asyncio_utils from mitmproxy.utils import human from mitmproxy.utils.data import pkg_data @@ -414,7 +413,7 @@ def request(flow: http.HTTPFlow): if "redirect" in flow.request.path: flow.request.host = "httpbin.org" - def tls_start_client(tls_start: tls.TlsHookData): + def tls_start_client(tls_start: tls.TlsData): # INSECURE ssl_context = SSL.Context(SSL.SSLv23_METHOD) ssl_context.use_privatekey_file( @@ -426,7 +425,7 @@ def tls_start_client(tls_start: tls.TlsHookData): tls_start.ssl_conn = SSL.Connection(ssl_context) tls_start.ssl_conn.set_accept_state() - def tls_start_server(tls_start: tls.TlsHookData): + def tls_start_server(tls_start: tls.TlsData): # INSECURE ssl_context = SSL.Context(SSL.SSLv23_METHOD) tls_start.ssl_conn = SSL.Connection(ssl_context) diff --git a/mitmproxy/tls.py b/mitmproxy/tls.py index 4e95f231dd..55b14668bb 100644 --- a/mitmproxy/tls.py +++ b/mitmproxy/tls.py @@ -1,13 +1,20 @@ import io +from dataclasses import dataclass from typing import List, Optional, Tuple from kaitaistruct import KaitaiStream +from OpenSSL import SSL +from mitmproxy import connection from mitmproxy.contrib.kaitaistruct import tls_client_hello from mitmproxy.net import check +from mitmproxy.proxy import context class ClientHello: + """ + A TLS ClientHello is the first message sent by the client when initiating TLS. + """ def __init__(self, raw_client_hello): self._client_hello = tls_client_hello.TlsClientHello( @@ -23,10 +30,10 @@ def sni(self) -> Optional[str]: if self._client_hello.extensions: for extension in self._client_hello.extensions.extensions: is_valid_sni_extension = ( - extension.type == 0x00 and - len(extension.body.server_names) == 1 and - extension.body.server_names[0].name_type == 0 and - check.is_valid_host(extension.body.server_names[0].host_name) + extension.type == 0x00 and + len(extension.body.server_names) == 1 and + extension.body.server_names[0].name_type == 0 and + check.is_valid_host(extension.body.server_names[0].host_name) ) if is_valid_sni_extension: return extension.body.server_names[0].host_name.decode("ascii") @@ -51,3 +58,39 @@ def extensions(self) -> List[Tuple[int, bytes]]: def __repr__(self): return f"ClientHello(sni: {self.sni}, alpn_protocols: {self.alpn_protocols})" + + +@dataclass +class ClientHelloData: + """ + Event data for `tls_clienthello` event hooks. + """ + context: context.Context + """The context object for this connection.""" + client_hello: ClientHello + """The entire parsed TLS ClientHello.""" + ignore_connection: bool = False + """ + If set to `True`, do not intercept this connection and forward encrypted contents unmodified. + """ + establish_server_tls_first: bool = False + """ + If set to `True`, pause this handshake and establish TLS with an upstream server first. + This makes it possible to process the server certificate when generating an interception certificate. + """ + + +@dataclass +class TlsData: + """ + Event data for `tls_start_client`, `tls_start_server`, and `tls_handshake` event hooks. + """ + conn: connection.Connection + """The affected connection.""" + context: context.Context + """The context object for this connection.""" + ssl_conn: Optional[SSL.Connection] = None + """ + The associated pyOpenSSL `SSL.Connection` object. + This will be set by an addon in the `tls_start_*` event hooks. + """ diff --git a/test/mitmproxy/addons/test_tlsconfig.py b/test/mitmproxy/addons/test_tlsconfig.py index 268fd355ff..115d92c0ce 100644 --- a/test/mitmproxy/addons/test_tlsconfig.py +++ b/test/mitmproxy/addons/test_tlsconfig.py @@ -6,10 +6,10 @@ import pytest from OpenSSL import SSL -from mitmproxy import certs, connection +from mitmproxy import certs, connection, tls from mitmproxy.addons import tlsconfig from mitmproxy.proxy import context -from mitmproxy.proxy.layers import modes, tls +from mitmproxy.proxy.layers import modes, tls as proxy_tls from mitmproxy.test import taddons from test.mitmproxy.proxy.layers import test_tls @@ -130,7 +130,7 @@ def test_tls_start_client(self, tdata): ) ctx = context.Context(connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), tctx.options) - tls_start = tls.TlsHookData(ctx.client, context=ctx) + tls_start = tls.TlsData(ctx.client, context=ctx) ta.tls_start_client(tls_start) tssl_server = tls_start.ssl_conn tssl_client = test_tls.SSLTest() @@ -145,7 +145,7 @@ def test_tls_start_server_verify_failed(self): ctx.client.cipher_list = ["TLS_AES_256_GCM_SHA384", "ECDHE-RSA-AES128-SHA"] ctx.server.address = ("example.mitmproxy.org", 443) - tls_start = tls.TlsHookData(ctx.server, context=ctx) + tls_start = tls.TlsData(ctx.server, context=ctx) ta.tls_start_server(tls_start) tssl_client = tls_start.ssl_conn tssl_server = test_tls.SSLTest(server_side=True) @@ -160,7 +160,7 @@ def test_tls_start_server_verify_ok(self, tdata): tctx.configure(ta, ssl_verify_upstream_trusted_ca=tdata.path( "mitmproxy/net/data/verificationcerts/trusted-root.crt")) - tls_start = tls.TlsHookData(ctx.server, context=ctx) + tls_start = tls.TlsData(ctx.server, context=ctx) ta.tls_start_server(tls_start) tssl_client = tls_start.ssl_conn tssl_server = test_tls.SSLTest(server_side=True) @@ -179,7 +179,7 @@ def test_tls_start_server_insecure(self): http2=False, ciphers_server="ALL" ) - tls_start = tls.TlsHookData(ctx.server, context=ctx) + tls_start = tls.TlsData(ctx.server, context=ctx) ta.tls_start_server(tls_start) tssl_client = tls_start.ssl_conn tssl_server = test_tls.SSLTest(server_side=True) @@ -190,7 +190,7 @@ def test_alpn_selection(self): with taddons.context(ta) as tctx: ctx = context.Context(connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), tctx.options) ctx.server.address = ("example.mitmproxy.org", 443) - tls_start = tls.TlsHookData(ctx.server, context=ctx) + tls_start = tls.TlsData(ctx.server, context=ctx) def assert_alpn(http2, client_offers, expected): tctx.configure(ta, http2=http2) @@ -199,8 +199,8 @@ def assert_alpn(http2, client_offers, expected): ta.tls_start_server(tls_start) assert ctx.server.alpn_offers == expected - assert_alpn(True, tls.HTTP_ALPNS + (b"foo",), tls.HTTP_ALPNS + (b"foo",)) - assert_alpn(False, tls.HTTP_ALPNS + (b"foo",), tls.HTTP1_ALPNS + (b"foo",)) + assert_alpn(True, proxy_tls.HTTP_ALPNS + (b"foo",), proxy_tls.HTTP_ALPNS + (b"foo",)) + assert_alpn(False, proxy_tls.HTTP_ALPNS + (b"foo",), proxy_tls.HTTP1_ALPNS + (b"foo",)) assert_alpn(True, [], []) assert_alpn(False, [], []) ctx.client.timestamp_tls_setup = time.time() @@ -222,7 +222,7 @@ def test_no_h2_proxy(self, tdata): modes.HttpProxy(ctx), 123 ] - tls_start = tls.TlsHookData(ctx.client, context=ctx) + tls_start = tls.TlsData(ctx.client, context=ctx) ta.tls_start_client(tls_start) assert tls_start.ssl_conn.get_app_data()["client_alpn"] == b"http/1.1" @@ -244,7 +244,7 @@ def test_client_cert_file(self, tdata, client_certs): ssl_verify_upstream_trusted_ca=tdata.path("mitmproxy/net/data/verificationcerts/trusted-root.crt"), ) - tls_start = tls.TlsHookData(ctx.server, context=ctx) + tls_start = tls.TlsData(ctx.server, context=ctx) ta.tls_start_server(tls_start) tssl_client = tls_start.ssl_conn tssl_server = test_tls.SSLTest(server_side=True) diff --git a/test/mitmproxy/proxy/layers/test_tls.py b/test/mitmproxy/proxy/layers/test_tls.py index 4c4aad7d15..e20537c34c 100644 --- a/test/mitmproxy/proxy/layers/test_tls.py +++ b/test/mitmproxy/proxy/layers/test_tls.py @@ -9,6 +9,7 @@ from mitmproxy.connection import ConnectionState, Server from mitmproxy.proxy import commands, context, events, layer from mitmproxy.proxy.layers import tls +from mitmproxy.tls import ClientHelloData, TlsData from mitmproxy.utils import data from test.mitmproxy.proxy import tutils @@ -68,8 +69,8 @@ def test_get_client_hello(): assert tls.get_client_hello(single_record) == client_hello_no_extensions split_over_two_records = ( - bytes.fromhex("1603010020") + client_hello_no_extensions[:32] + - bytes.fromhex("1603010045") + client_hello_no_extensions[32:] + bytes.fromhex("1603010020") + client_hello_no_extensions[:32] + + bytes.fromhex("1603010045") + client_hello_no_extensions[32:] ) assert tls.get_client_hello(split_over_two_records) == client_hello_no_extensions @@ -134,9 +135,9 @@ def _test_echo(playbook: tutils.Playbook, tssl: SSLTest, conn: connection.Connec tssl.obj.write(b"Hello World") data = tutils.Placeholder(bytes) assert ( - playbook - >> events.DataReceived(conn, tssl.bio_read()) - << commands.SendData(conn, data) + playbook + >> events.DataReceived(conn, tssl.bio_read()) + << commands.SendData(conn, data) ) tssl.bio_write(data()) assert tssl.obj.read() == b"hello world" @@ -156,13 +157,13 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: def finish_handshake(playbook: tutils.Playbook, conn: connection.Connection, tssl: SSLTest): data = tutils.Placeholder(bytes) - tls_hook_data = tutils.Placeholder(tls.TlsHookData) + tls_hook_data = tutils.Placeholder(TlsData) assert ( - playbook - >> events.DataReceived(conn, tssl.bio_read()) - << tls.TlsHandshakeHook(tls_hook_data) - >> tutils.reply() - << commands.SendData(conn, data) + playbook + >> events.DataReceived(conn, tssl.bio_read()) + << tls.TlsHandshakeHook(tls_hook_data) + >> tutils.reply() + << commands.SendData(conn, data) ) assert tls_hook_data().conn.error is None tssl.bio_write(data()) @@ -173,7 +174,7 @@ def reply_tls_start_client(alpn: typing.Optional[bytes] = None, *args, **kwargs) Helper function to simplify the syntax for tls_start_client hooks. """ - def make_client_conn(tls_start: tls.TlsHookData) -> None: + def make_client_conn(tls_start: TlsData) -> None: # ssl_context = SSL.Context(Method.TLS_METHOD) # ssl_context.set_min_proto_version(SSL.TLS1_3_VERSION) ssl_context = SSL.Context(SSL.SSLv23_METHOD) @@ -198,7 +199,7 @@ def reply_tls_start_server(alpn: typing.Optional[bytes] = None, *args, **kwargs) Helper function to simplify the syntax for tls_start_server hooks. """ - def make_server_conn(tls_start: tls.TlsHookData) -> None: + def make_server_conn(tls_start: TlsData) -> None: # ssl_context = SSL.Context(Method.TLS_METHOD) # ssl_context.set_min_proto_version(SSL.TLS1_3_VERSION) ssl_context = SSL.Context(SSL.SSLv23_METHOD) @@ -243,9 +244,9 @@ def test_not_connected(self, tctx: context.Context): layer.child_layer = TlsEchoLayer(tctx) assert ( - tutils.Playbook(layer) - >> events.DataReceived(tctx.client, b"Hello World") - << commands.SendData(tctx.client, b"hello world") + tutils.Playbook(layer) + >> events.DataReceived(tctx.client, b"Hello World") + << commands.SendData(tctx.client, b"hello world") ) def test_simple(self, tctx): @@ -259,10 +260,10 @@ def test_simple(self, tctx): # send ClientHello, receive ClientHello data = tutils.Placeholder(bytes) assert ( - playbook - << tls.TlsStartServerHook(tutils.Placeholder()) - >> reply_tls_start_server() - << commands.SendData(tctx.server, data) + playbook + << tls.TlsStartServerHook(tutils.Placeholder()) + >> reply_tls_start_server() + << commands.SendData(tctx.server, data) ) tssl.bio_write(data()) with pytest.raises(ssl.SSLWantReadError): @@ -274,31 +275,31 @@ def test_simple(self, tctx): # finish handshake (locally) tssl.do_handshake() assert ( - playbook - >> events.DataReceived(tctx.server, tssl.bio_read()) - << None + playbook + >> events.DataReceived(tctx.server, tssl.bio_read()) + << None ) assert tctx.server.tls_established # Echo assert ( - playbook - >> events.DataReceived(tctx.client, b"foo") - << layer.NextLayerHook(tutils.Placeholder()) - >> tutils.reply_next_layer(TlsEchoLayer) - << commands.SendData(tctx.client, b"foo") + playbook + >> events.DataReceived(tctx.client, b"foo") + << layer.NextLayerHook(tutils.Placeholder()) + >> tutils.reply_next_layer(TlsEchoLayer) + << commands.SendData(tctx.client, b"foo") ) _test_echo(playbook, tssl, tctx.server) with pytest.raises(ssl.SSLWantReadError): tssl.obj.unwrap() assert ( - playbook - >> events.DataReceived(tctx.server, tssl.bio_read()) - << commands.CloseConnection(tctx.server) - >> events.ConnectionClosed(tctx.server) - << None + playbook + >> events.DataReceived(tctx.server, tssl.bio_read()) + << commands.CloseConnection(tctx.server) + >> events.ConnectionClosed(tctx.server) + << None ) def test_untrusted_cert(self, tctx): @@ -312,15 +313,15 @@ def test_untrusted_cert(self, tctx): # send ClientHello data = tutils.Placeholder(bytes) assert ( - playbook - >> events.DataReceived(tctx.client, b"open-connection") - << layer.NextLayerHook(tutils.Placeholder()) - >> tutils.reply_next_layer(TlsEchoLayer) - << commands.OpenConnection(tctx.server) - >> tutils.reply(None) - << tls.TlsStartServerHook(tutils.Placeholder()) - >> reply_tls_start_server() - << commands.SendData(tctx.server, data) + playbook + >> events.DataReceived(tctx.client, b"open-connection") + << layer.NextLayerHook(tutils.Placeholder()) + >> tutils.reply_next_layer(TlsEchoLayer) + << commands.OpenConnection(tctx.server) + >> tutils.reply(None) + << tls.TlsStartServerHook(tutils.Placeholder()) + >> reply_tls_start_server() + << commands.SendData(tctx.server, data) ) # receive ServerHello, finish client handshake @@ -328,16 +329,16 @@ def test_untrusted_cert(self, tctx): with pytest.raises(ssl.SSLWantReadError): tssl.do_handshake() - tls_hook_data = tutils.Placeholder(tls.TlsHookData) + tls_hook_data = tutils.Placeholder(TlsData) assert ( - playbook - >> events.DataReceived(tctx.server, tssl.bio_read()) - << commands.Log("Server TLS handshake failed. Certificate verify failed: Hostname mismatch", "warn") - << tls.TlsHandshakeHook(tls_hook_data) - >> tutils.reply() - << commands.CloseConnection(tctx.server) - << commands.SendData(tctx.client, - b"open-connection failed: Certificate verify failed: Hostname mismatch") + playbook + >> events.DataReceived(tctx.server, tssl.bio_read()) + << commands.Log("Server TLS handshake failed. Certificate verify failed: Hostname mismatch", "warn") + << tls.TlsHandshakeHook(tls_hook_data) + >> tutils.reply() + << commands.CloseConnection(tctx.server) + << commands.SendData(tctx.client, + b"open-connection failed: Certificate verify failed: Hostname mismatch") ) assert tls_hook_data().conn.error == "Certificate verify failed: Hostname mismatch" assert not tctx.server.tls_established @@ -349,17 +350,17 @@ def test_remote_speaks_no_tls(self, tctx): # send ClientHello, receive random garbage back data = tutils.Placeholder(bytes) - tls_hook_data = tutils.Placeholder(tls.TlsHookData) + tls_hook_data = tutils.Placeholder(TlsData) assert ( - playbook - << tls.TlsStartServerHook(tutils.Placeholder()) - >> reply_tls_start_server() - << commands.SendData(tctx.server, data) - >> events.DataReceived(tctx.server, b"HTTP/1.1 404 Not Found\r\n") - << commands.Log("Server TLS handshake failed. The remote server does not speak TLS.", "warn") - << tls.TlsHandshakeHook(tls_hook_data) - >> tutils.reply() - << commands.CloseConnection(tctx.server) + playbook + << tls.TlsStartServerHook(tutils.Placeholder()) + >> reply_tls_start_server() + << commands.SendData(tctx.server, data) + >> events.DataReceived(tctx.server, b"HTTP/1.1 404 Not Found\r\n") + << commands.Log("Server TLS handshake failed. The remote server does not speak TLS.", "warn") + << tls.TlsHandshakeHook(tls_hook_data) + >> tutils.reply() + << commands.CloseConnection(tctx.server) ) assert tls_hook_data().conn.error == "The remote server does not speak TLS." @@ -388,7 +389,7 @@ def test_unsupported_protocol(self, tctx: context.Context): tssl.do_handshake() # send back error - tls_hook_data = tutils.Placeholder(tls.TlsHookData) + tls_hook_data = tutils.Placeholder(TlsData) assert ( playbook >> events.DataReceived(tctx.server, tssl.bio_read()) @@ -402,8 +403,8 @@ def test_unsupported_protocol(self, tctx: context.Context): def make_client_tls_layer( - tctx: context.Context, - **kwargs + tctx: context.Context, + **kwargs ) -> typing.Tuple[tutils.Playbook, tls.ClientTLSLayer, SSLTest]: # This is a bit contrived as the client layer expects a server layer as parent. # We also set child layers manually to avoid NextLayer noise. @@ -435,13 +436,13 @@ def test_client_only(self, tctx: context.Context): # Send ClientHello, receive ServerHello data = tutils.Placeholder(bytes) assert ( - playbook - >> events.DataReceived(tctx.client, tssl_client.bio_read()) - << tls.TlsClienthelloHook(tutils.Placeholder()) - >> tutils.reply() - << tls.TlsStartClientHook(tutils.Placeholder()) - >> reply_tls_start_client() - << commands.SendData(tctx.client, data) + playbook + >> events.DataReceived(tctx.client, tssl_client.bio_read()) + << tls.TlsClienthelloHook(tutils.Placeholder()) + >> tutils.reply() + << tls.TlsStartClientHook(tutils.Placeholder()) + >> reply_tls_start_client() + << commands.SendData(tctx.client, data) ) tssl_client.bio_write(data()) tssl_client.do_handshake() @@ -455,9 +456,9 @@ def test_client_only(self, tctx: context.Context): _test_echo(playbook, tssl_client, tctx.client) other_server = Server(None) assert ( - playbook - >> events.DataReceived(other_server, b"Plaintext") - << commands.SendData(other_server, b"plaintext") + playbook + >> events.DataReceived(other_server, b"Plaintext") + << commands.SendData(other_server, b"plaintext") ) @pytest.mark.parametrize("server_state", ["open", "closed"]) @@ -474,14 +475,14 @@ def test_server_required(self, tctx, server_state): # We should now get instructed to open a server connection. data = tutils.Placeholder(bytes) - def require_server_conn(client_hello: tls.ClientHelloData) -> None: + def require_server_conn(client_hello: ClientHelloData) -> None: client_hello.establish_server_tls_first = True ( - playbook - >> events.DataReceived(tctx.client, tssl_client.bio_read()) - << tls.TlsClienthelloHook(tutils.Placeholder()) - >> tutils.reply(side_effect=require_server_conn) + playbook + >> events.DataReceived(tctx.client, tssl_client.bio_read()) + << tls.TlsClienthelloHook(tutils.Placeholder()) + >> tutils.reply(side_effect=require_server_conn) ) if server_state == "closed": ( @@ -503,12 +504,12 @@ def require_server_conn(client_hello: tls.ClientHelloData) -> None: data = tutils.Placeholder(bytes) assert ( - playbook - >> events.DataReceived(tctx.server, tssl_server.bio_read()) - << tls.TlsHandshakeHook(tutils.Placeholder()) - >> tutils.reply() - << commands.SendData(tctx.server, data) - << tls.TlsStartClientHook(tutils.Placeholder()) + playbook + >> events.DataReceived(tctx.server, tssl_server.bio_read()) + << tls.TlsHandshakeHook(tutils.Placeholder()) + >> tutils.reply() + << commands.SendData(tctx.server, data) + << tls.TlsStartClientHook(tutils.Placeholder()) ) tssl_server.bio_write(data()) assert tctx.server.tls_established @@ -516,9 +517,9 @@ def require_server_conn(client_hello: tls.ClientHelloData) -> None: data = tutils.Placeholder(bytes) assert ( - playbook - >> reply_tls_start_client(alpn=b"quux") - << commands.SendData(tctx.client, data) + playbook + >> reply_tls_start_client(alpn=b"quux") + << commands.SendData(tctx.client, data) ) tssl_client.bio_write(data()) tssl_client.do_handshake() @@ -544,7 +545,7 @@ def test_passthrough_from_clienthello(self, tctx, server_state): playbook, client_layer, tssl_client = make_client_tls_layer(tctx, alpn=["quux"]) - def make_passthrough(client_hello: tls.ClientHelloData) -> None: + def make_passthrough(client_hello: ClientHelloData) -> None: client_hello.ignore_connection = True client_hello = tssl_client.bio_read() @@ -570,17 +571,17 @@ def make_passthrough(client_hello: tls.ClientHelloData) -> None: def test_cannot_parse_clienthello(self, tctx: context.Context): """Test the scenario where we cannot parse the ClientHello""" playbook, client_layer, tssl_client = make_client_tls_layer(tctx) - tls_hook_data = tutils.Placeholder(tls.TlsHookData) + tls_hook_data = tutils.Placeholder(TlsData) invalid = b"\x16\x03\x01\x00\x00" assert ( - playbook - >> events.DataReceived(tctx.client, invalid) - << commands.Log(f"Client TLS handshake failed. Cannot parse ClientHello: {invalid.hex()}", level="warn") - << tls.TlsHandshakeHook(tls_hook_data) - >> tutils.reply() - << commands.CloseConnection(tctx.client) + playbook + >> events.DataReceived(tctx.client, invalid) + << commands.Log(f"Client TLS handshake failed. Cannot parse ClientHello: {invalid.hex()}", level="warn") + << tls.TlsHandshakeHook(tls_hook_data) + >> tutils.reply() + << commands.CloseConnection(tctx.client) ) assert tls_hook_data().conn.error assert not tctx.client.tls_established @@ -601,28 +602,28 @@ def test_mitmproxy_ca_is_untrusted(self, tctx: context.Context): data = tutils.Placeholder(bytes) assert ( - playbook - >> events.DataReceived(tctx.client, tssl_client.bio_read()) - << tls.TlsClienthelloHook(tutils.Placeholder()) - >> tutils.reply() - << tls.TlsStartClientHook(tutils.Placeholder()) - >> reply_tls_start_client() - << commands.SendData(tctx.client, data) + playbook + >> events.DataReceived(tctx.client, tssl_client.bio_read()) + << tls.TlsClienthelloHook(tutils.Placeholder()) + >> tutils.reply() + << tls.TlsStartClientHook(tutils.Placeholder()) + >> reply_tls_start_client() + << commands.SendData(tctx.client, data) ) tssl_client.bio_write(data()) with pytest.raises(ssl.SSLCertVerificationError): tssl_client.do_handshake() # Finish Handshake - tls_hook_data = tutils.Placeholder(tls.TlsHookData) + tls_hook_data = tutils.Placeholder(TlsData) assert ( - playbook - >> events.DataReceived(tctx.client, tssl_client.bio_read()) - << commands.Log("Client TLS handshake failed. The client does not trust the proxy's certificate " - "for wrong.host.mitmproxy.org (sslv3 alert bad certificate)", "warn") - << tls.TlsHandshakeHook(tls_hook_data) - >> tutils.reply() - << commands.CloseConnection(tctx.client) - >> events.ConnectionClosed(tctx.client) + playbook + >> events.DataReceived(tctx.client, tssl_client.bio_read()) + << commands.Log("Client TLS handshake failed. The client does not trust the proxy's certificate " + "for wrong.host.mitmproxy.org (sslv3 alert bad certificate)", "warn") + << tls.TlsHandshakeHook(tls_hook_data) + >> tutils.reply() + << commands.CloseConnection(tctx.client) + >> events.ConnectionClosed(tctx.client) ) assert not tctx.client.tls_established assert tls_hook_data().conn.error @@ -634,7 +635,7 @@ def test_immediate_disconnect(self, tctx: context.Context, close_at): the proxy certificate.""" playbook, client_layer, tssl_client = make_client_tls_layer(tctx, sni=b"wrong.host.mitmproxy.org") playbook.logs = True - tls_hook_data = tutils.Placeholder(tls.TlsHookData) + tls_hook_data = tutils.Placeholder(TlsData) playbook >> events.DataReceived(tctx.client, tssl_client.bio_read()) playbook << tls.TlsClienthelloHook(tutils.Placeholder()) @@ -687,7 +688,7 @@ def test_unsupported_protocol(self, tctx: context.Context): playbook, client_layer, tssl_client = make_client_tls_layer(tctx, max_ver=ssl.TLSVersion.TLSv1_2) playbook.logs = True - tls_hook_data = tutils.Placeholder(tls.TlsHookData) + tls_hook_data = tutils.Placeholder(TlsData) assert ( playbook >> events.DataReceived(tctx.client, tssl_client.bio_read()) From 017344dfe48fc4fc2a88d55740814b9067d18e88 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 4 Sep 2021 17:03:26 +0200 Subject: [PATCH 005/285] tls: api docs++ --- docs/scripts/api-render.py | 2 ++ docs/scripts/pdoc-template/module.html.jinja2 | 8 ++++++++ docs/src/content/api/mitmproxy.proxy.context.md | 11 +++++++++++ docs/src/content/api/mitmproxy.tls.md | 11 +++++++++++ mitmproxy/proxy/context.py | 15 ++++++++++++++- mitmproxy/tls.py | 13 ++++++++++++- 6 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 docs/src/content/api/mitmproxy.proxy.context.md create mode 100644 docs/src/content/api/mitmproxy.tls.md diff --git a/docs/scripts/api-render.py b/docs/scripts/api-render.py index f7c58d091a..a293db7ddd 100644 --- a/docs/scripts/api-render.py +++ b/docs/scripts/api-render.py @@ -30,8 +30,10 @@ "mitmproxy.flow", "mitmproxy.http", "mitmproxy.net.server_spec", + "mitmproxy.proxy.context", "mitmproxy.proxy.server_hooks", "mitmproxy.tcp", + "mitmproxy.tls", "mitmproxy.websocket", here / ".." / "src" / "generated" / "events.py", ] diff --git a/docs/scripts/pdoc-template/module.html.jinja2 b/docs/scripts/pdoc-template/module.html.jinja2 index 268c393fb8..d6d731c246 100644 --- a/docs/scripts/pdoc-template/module.html.jinja2 +++ b/docs/scripts/pdoc-template/module.html.jinja2 @@ -55,6 +55,14 @@ To document all event hooks, we do a bit of hackery: {% if doc.qualname.startswith("ServerConnectionHookData") and doc.name != "__init__" %} {{ default_is_public(doc) }} {% endif %} + {% elif doc.modulename == "mitmproxy.proxy.context" %} + {% if doc.qualname is not in(["Context.__init__", "Context.fork", "Context.options"]) %} + {{ default_is_public(doc) }} + {% endif %} + {% elif doc.modulename == "mitmproxy.tls" %} + {% if doc.qualname is not in(["TlsData.__init__", "ClientHelloData.__init__"]) %} + {{ default_is_public(doc) }} + {% endif %} {% elif doc.modulename == "mitmproxy.websocket" %} {% if doc.qualname != "WebSocketMessage.type" %} {{ default_is_public(doc) }} diff --git a/docs/src/content/api/mitmproxy.proxy.context.md b/docs/src/content/api/mitmproxy.proxy.context.md new file mode 100644 index 0000000000..b4aa488654 --- /dev/null +++ b/docs/src/content/api/mitmproxy.proxy.context.md @@ -0,0 +1,11 @@ + +--- +title: "mitmproxy.proxy.context" +url: "api/mitmproxy/proxy/context.html" + +menu: + addons: + parent: 'Event Hooks & API' +--- + +{{< readfile file="/generated/api/mitmproxy/proxy/context.html" >}} diff --git a/docs/src/content/api/mitmproxy.tls.md b/docs/src/content/api/mitmproxy.tls.md new file mode 100644 index 0000000000..02dcbeb30e --- /dev/null +++ b/docs/src/content/api/mitmproxy.tls.md @@ -0,0 +1,11 @@ + +--- +title: "mitmproxy.tls" +url: "api/mitmproxy/tls.html" + +menu: + addons: + parent: 'Event Hooks & API' +--- + +{{< readfile file="/generated/api/mitmproxy/tls.html" >}} diff --git a/mitmproxy/proxy/context.py b/mitmproxy/proxy/context.py index 0114c00c50..8f8025b9fa 100644 --- a/mitmproxy/proxy/context.py +++ b/mitmproxy/proxy/context.py @@ -9,13 +9,26 @@ class Context: """ - The context object provided to each `mitmproxy.proxy.layer.Layer` by its parent layer. + The context object provided to each protocol layer in the proxy core. """ client: connection.Client + """The client connection.""" server: connection.Server + """ + The server connection. + + For practical reasons this attribute is always set, even if there is not server connection yet. + In this case the server address is `None`. + """ options: Options + """ + Provides access to options for proxy layers. Not intended for use by addons, use `mitmproxy.ctx.options` instead. + """ layers: List["mitmproxy.proxy.layer.Layer"] + """ + The protocol layer stack. + """ def __init__( self, diff --git a/mitmproxy/tls.py b/mitmproxy/tls.py index 55b14668bb..31b6071076 100644 --- a/mitmproxy/tls.py +++ b/mitmproxy/tls.py @@ -16,17 +16,23 @@ class ClientHello: A TLS ClientHello is the first message sent by the client when initiating TLS. """ - def __init__(self, raw_client_hello): + def __init__(self, raw_client_hello: bytes): + """Create a TLS ClientHello object from raw bytes.""" self._client_hello = tls_client_hello.TlsClientHello( KaitaiStream(io.BytesIO(raw_client_hello)) ) @property def cipher_suites(self) -> List[int]: + """The cipher suites offered by the client (as raw ints).""" return self._client_hello.cipher_suites.cipher_suites @property def sni(self) -> Optional[str]: + """ + The [Server Name Indication](https://en.wikipedia.org/wiki/Server_Name_Indication), + which indicates which hostname the client wants to connect to. + """ if self._client_hello.extensions: for extension in self._client_hello.extensions.extensions: is_valid_sni_extension = ( @@ -41,6 +47,10 @@ def sni(self) -> Optional[str]: @property def alpn_protocols(self) -> List[bytes]: + """ + The application layer protocols offered by the client as part of the + [ALPN](https://en.wikipedia.org/wiki/Application-Layer_Protocol_Negotiation) TLS extension. + """ if self._client_hello.extensions: for extension in self._client_hello.extensions.extensions: if extension.type == 0x10: @@ -49,6 +59,7 @@ def alpn_protocols(self) -> List[bytes]: @property def extensions(self) -> List[Tuple[int, bytes]]: + """The raw list of extensions in the form of `(extension_type, raw_bytes)` tuples.""" ret = [] if self._client_hello.extensions: for extension in self._client_hello.extensions.extensions: From fd5caf40c75ca73c4b767170497abf6a5bf016a0 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 5 Nov 2021 18:48:12 +0100 Subject: [PATCH 006/285] server replay: improve option help (#4894) --- mitmproxy/addons/serverplayback.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/mitmproxy/addons/serverplayback.py b/mitmproxy/addons/serverplayback.py index 149a7c0a26..144ec6b9e1 100644 --- a/mitmproxy/addons/serverplayback.py +++ b/mitmproxy/addons/serverplayback.py @@ -21,7 +21,7 @@ def __init__(self): def load(self, loader): loader.add_option( "server_replay_kill_extra", bool, False, - "Kill extra requests during replay." + "Kill extra requests during replay (for which no replayable response was found)." ) loader.add_option( "server_replay_nopop", bool, False, @@ -39,7 +39,10 @@ def load(self, loader): ) loader.add_option( "server_replay_use_headers", typing.Sequence[str], [], - "Request headers to be considered during replay." + """ + Request headers that need to match while searching for a saved flow + to replay. + """ ) loader.add_option( "server_replay", typing.Sequence[str], [], @@ -47,19 +50,19 @@ def load(self, loader): ) loader.add_option( "server_replay_ignore_content", bool, False, - "Ignore request's content while searching for a saved flow to replay." + "Ignore request content while searching for a saved flow to replay." ) loader.add_option( "server_replay_ignore_params", typing.Sequence[str], [], """ - Request's parameters to be ignored while searching for a saved flow + Request parameters to be ignored while searching for a saved flow to replay. """ ) loader.add_option( "server_replay_ignore_payload_params", typing.Sequence[str], [], """ - Request's payload parameters (application/x-www-form-urlencoded or + Request payload parameters (application/x-www-form-urlencoded or multipart/form-data) to be ignored while searching for a saved flow to replay. """ @@ -67,14 +70,14 @@ def load(self, loader): loader.add_option( "server_replay_ignore_host", bool, False, """ - Ignore request's destination host while searching for a saved flow + Ignore request destination host while searching for a saved flow to replay. """ ) loader.add_option( "server_replay_ignore_port", bool, False, """ - Ignore request's destination port while searching for a saved flow + Ignore request destination port while searching for a saved flow to replay. """ ) From df32d610869c73abf0840e148a0d7cdc64b1f61e Mon Sep 17 00:00:00 2001 From: Karl Parkinson Date: Thu, 11 Nov 2021 01:37:00 -0700 Subject: [PATCH 007/285] Remove pyopenssl cruft (#4897) * remove old pyopenssl cruft * bump minimum version of pyopenssl * add extra spaces to conform to style guide * update changelog * replace getattr with direct SSL method calls * put version check back in but remove setdefault method calls * tweak changelog wording * bumb tox.ini pyOpenSSL dependency version Co-authored-by: Karl Parkinson --- CHANGELOG.md | 1 + mitmproxy/net/tls.py | 20 +++++++------------- setup.py | 2 +- tox.ini | 2 +- 4 files changed, 10 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7211e99b3..8db2dd7712 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ * Fix a crash caused when editing string option (#4852, @rbdixon) * Base container image bumped to Debian 11 Bullseye (@Kriechi) * Upstream replays don't do CONNECT on plaintext HTTP requests (#4876, @HoffmannP) +* Remove workarounds for old pyOpenSSL versions (#4831, @KarlParkinson) ## 28 September 2021: mitmproxy 7.0.4 diff --git a/mitmproxy/net/tls.py b/mitmproxy/net/tls.py index 7e85fd55e9..35532675b6 100644 --- a/mitmproxy/net/tls.py +++ b/mitmproxy/net/tls.py @@ -21,29 +21,23 @@ # redeclared here for strict type checking class Method(Enum): - # TODO: just SSL attributes once https://github.com/pyca/pyopenssl/pull/985 has landed. - TLS_SERVER_METHOD = getattr(SSL, "TLS_SERVER_METHOD", 8) - TLS_CLIENT_METHOD = getattr(SSL, "TLS_CLIENT_METHOD", 9) + TLS_SERVER_METHOD = SSL.TLS_SERVER_METHOD + TLS_CLIENT_METHOD = SSL.TLS_CLIENT_METHOD -# TODO: remove once https://github.com/pyca/pyopenssl/pull/985 has landed. try: SSL._lib.TLS_server_method # type: ignore except AttributeError as e: # pragma: no cover raise RuntimeError("Your installation of the cryptography Python package is outdated.") from e -SSL.Context._methods.setdefault(Method.TLS_SERVER_METHOD.value, SSL._lib.TLS_server_method) # type: ignore -SSL.Context._methods.setdefault(Method.TLS_CLIENT_METHOD.value, SSL._lib.TLS_client_method) # type: ignore - class Version(Enum): UNBOUNDED = 0 - # TODO: just SSL attributes once https://github.com/pyca/pyopenssl/pull/985 has landed. - SSL3 = getattr(SSL, "SSL3_VERSION", 768) - TLS1 = getattr(SSL, "TLS1_VERSION", 769) - TLS1_1 = getattr(SSL, "TLS1_1_VERSION", 770) - TLS1_2 = getattr(SSL, "TLS1_2_VERSION", 771) - TLS1_3 = getattr(SSL, "TLS1_3_VERSION", 772) + SSL3 = SSL.SSL3_VERSION + TLS1 = SSL.TLS1_VERSION + TLS1_1 = SSL.TLS1_1_VERSION + TLS1_2 = SSL.TLS1_2_VERSION + TLS1_3 = SSL.TLS1_3_VERSION class Verify(Enum): diff --git a/setup.py b/setup.py index ff52eef9c9..59ef56389b 100644 --- a/setup.py +++ b/setup.py @@ -82,7 +82,7 @@ "msgpack>=1.0.0, <1.1.0", "passlib>=1.6.5, <1.8", "protobuf>=3.14,<3.19", - "pyOpenSSL>=20.0,<21.1", + "pyOpenSSL>=21.0,<21.1", "pyparsing>=2.4.2,<2.5", "pyperclip>=1.6.0,<1.9", "ruamel.yaml>=0.16,<0.17.17", diff --git a/tox.ini b/tox.ini index 3fb39b90b5..cbd69e163c 100644 --- a/tox.ini +++ b/tox.ini @@ -35,7 +35,7 @@ deps = types-Werkzeug==1.0.5 types-requests==2.25.9 types-cryptography==3.3.5 - types-pyOpenSSL==20.0.6 + types-pyOpenSSL==21.0.0 commands = mypy {posargs} From 9571491e504ab644cfa979e8d91be0521a8c9726 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 16 Nov 2021 10:26:38 +0100 Subject: [PATCH 008/285] StackOverflow -> GitHub Discussions --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2ffcb4be78..a92e0b0b76 100644 --- a/README.md +++ b/README.md @@ -29,9 +29,9 @@ The documentation for mitmproxy is available on our website: [![mitmproxy documentation dev](https://shields.mitmproxy.org/badge/docs-dev-brightgreen.svg)](https://docs.mitmproxy.org/main/) If you have questions on how to use mitmproxy, please -ask them on StackOverflow! +use GitHub Discussions! -[![StackOverflow: mitmproxy](https://shields.mitmproxy.org/stackexchange/stackoverflow/t/mitmproxy?color=orange&label=stackoverflow%20questions)](https://stackoverflow.com/questions/tagged/mitmproxy) +[![mitmproxy discussions](https://shields.mitmproxy.org/badge/help-github%20discussions-orange.svg)](https://github.com/mitmproxy/mitmproxy/discussions) ## Contributing From 4f47612548682a1b13100cde99d3c067a34944b1 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 16 Nov 2021 10:26:38 +0100 Subject: [PATCH 009/285] StackOverflow -> GitHub Discussions (part 2) --- .github/ISSUE_TEMPLATE/config.yml | 5 ++--- CONTRIBUTING.md | 2 +- docs/src/content/overview-getting-started.md | 6 +++--- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 47db052bd8..c31551c123 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,6 +1,5 @@ blank_issues_enabled: false contact_links: - name: Support - url: https://stackoverflow.com/questions/tagged/mitmproxy - about: Please do not use GitHub for support requests. - If you have questions on how to use mitmproxy, please ask them on StackOverflow! + url: https://github.com/mitmproxy/mitmproxy/discussions + about: If you have questions on how to use mitmproxy, ask them on the discussions page! diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index df060f95d8..27a85695b5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,7 +5,7 @@ forward, please consider contributing in the following areas: - **Maintenance:** We are *incredibly* thankful for individuals who are stepping up and helping with maintenance. This includes (but is not limited to) triaging issues, reviewing pull requests and picking up stale ones, helping out other - users on [StackOverflow](https://stackoverflow.com/questions/tagged/mitmproxy), creating minimal, complete and + users on [GitHub Discussions](https://github.com/mitmproxy/mitmproxy/discussions), creating minimal, complete and verifiable examples or test cases for existing bug reports, updating documentation, or fixing minor bugs that have recently been reported. - **Code Contributions:** We actively mark issues that we consider are [good first contributions]( diff --git a/docs/src/content/overview-getting-started.md b/docs/src/content/overview-getting-started.md index 10a4aa7ddd..7bcfd63598 100644 --- a/docs/src/content/overview-getting-started.md +++ b/docs/src/content/overview-getting-started.md @@ -47,6 +47,6 @@ new flow and you can inspect it. ## Resources -* [**StackOverflow**](https://stackoverflow.com/questions/tagged/mitmproxy): If you want to ask usage questions, please do so on StackOverflow. -* [**GitHub**](https://github.com/mitmproxy/): If you want to contribute to mitmproxy or submit a bug report, please do so on GitHub. -* [**Slack**](https://mitmproxy.slack.com): If you want to get in touch with the developers or other users, please use our Slack channel. +* [**GitHub**](https://github.com/mitmproxy/mitmproxy): If you want to ask usage questions, contribute + to mitmproxy, or submit a bug report, please use GitHub. +* [**Slack**](https://mitmproxy.slack.com): For ephemeral development questions/coordination, please use our Slack channel. From 301f8cf79d1793826a9b73fca6406c005a1c1638 Mon Sep 17 00:00:00 2001 From: Marius Date: Tue, 16 Nov 2021 05:47:34 -0600 Subject: [PATCH 010/285] Create search.py (#4900) * Create search.py * Address linter findings * Address linter findings --- examples/contrib/search.py | 94 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 examples/contrib/search.py diff --git a/examples/contrib/search.py b/examples/contrib/search.py new file mode 100644 index 0000000000..76a4dff952 --- /dev/null +++ b/examples/contrib/search.py @@ -0,0 +1,94 @@ +import re +import typing + +from json import dumps + +from mitmproxy import command, ctx, flow + + +MARKER = ':mag:' +RESULTS_STR = 'Search Results: ' + + +class Search: + def __init__(self): + self.exp = None + + @command.command('search') + def _search(self, + flows: typing.Sequence[flow.Flow], + regex: str) -> None: + """ + Defines a command named "search" that matches + the given regular expression against most parts + of each request/response included in the selected flows. + + Usage: from the flow list view, type ":search" followed by + a space, then a flow selection expression; e.g., "@shown", + then the desired regular expression to perform the search. + + Alternatively, define a custom shortcut in keys.yaml; e.g.: + - + key: "/" + ctx: ["flowlist"] + cmd: "console.command search @shown " + + Flows containing matches to the expression will be marked + with the magnifying glass emoji, and their comments will + contain JSON-formatted search results. + + To view flow comments, enter the flow view + and navigate to the detail tab. + """ + + try: + self.exp = re.compile(regex) + except re.error as e: + ctx.log.error(e) + return + + for _flow in flows: + # Erase previous results while preserving other comments: + comments = list() + for c in _flow.comment.split('\n'): + if c.startswith(RESULTS_STR): + break + comments.append(c) + _flow.comment = '\n'.join(comments) + + if _flow.marked == MARKER: + _flow.marked = False + + results = {k: v for k, v in self.flow_results(_flow).items() if v} + if results: + comments.append(RESULTS_STR) + comments.append(dumps(results, indent=2)) + _flow.comment = '\n'.join(comments) + _flow.marked = MARKER + + def header_results(self, message): + results = {k: self.exp.findall(v) for k, v in message.headers.items()} + return {k: v for k, v in results.items() if v} + + def flow_results(self, _flow): + results = dict() + results.update( + {'flow_comment': self.exp.findall(_flow.comment)}) + if _flow.request is not None: + results.update( + {'request_path': self.exp.findall(_flow.request.path)}) + results.update( + {'request_headers': self.header_results(_flow.request)}) + if _flow.request.text: + results.update( + {'request_body': self.exp.findall(_flow.request.text)}) + if _flow.response is not None: + results.update( + {'response_headers': self.header_results(_flow.response)}) + if _flow.response.text: + results.update( + {'response_body': self.exp.findall(_flow.response.text)}) + return results + + +addons = [Search()] From eb43ae4709b94c8c4fd867d6133c3067394b1aa9 Mon Sep 17 00:00:00 2001 From: Christian Zenker Date: Tue, 16 Nov 2021 12:49:12 +0100 Subject: [PATCH 011/285] Use the slim base image for container (#4889) --- release/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/docker/Dockerfile b/release/docker/Dockerfile index 0f3d4ec036..9a0c7de327 100644 --- a/release/docker/Dockerfile +++ b/release/docker/Dockerfile @@ -4,7 +4,7 @@ ARG MITMPROXY_WHEEL COPY $MITMPROXY_WHEEL /wheels/ RUN pip install wheel && pip wheel --wheel-dir /wheels /wheels/${MITMPROXY_WHEEL} -FROM python:3.9-bullseye +FROM python:3.9-slim-bullseye RUN useradd -mU mitmproxy RUN apt-get update \ From 83f05897ce57dad5c98d787d23dc87ca4b0a663c Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 16 Nov 2021 21:10:48 +0100 Subject: [PATCH 012/285] fixup Windows transparent mode, fix #3876 and #3438 --- mitmproxy/addons/proxyserver.py | 4 ++-- mitmproxy/platform/windows.py | 12 +++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/mitmproxy/addons/proxyserver.py b/mitmproxy/addons/proxyserver.py index 3d9f258fea..dcf8ec32af 100644 --- a/mitmproxy/addons/proxyserver.py +++ b/mitmproxy/addons/proxyserver.py @@ -136,10 +136,10 @@ def configure(self, updated): except ValueError: raise exceptions.OptionsError(f"Invalid body_size_limit specification: " f"{ctx.options.body_size_limit}") - if not self.is_running: - return if "mode" in updated and ctx.options.mode == "transparent": # pragma: no cover platform.init_transparent_mode() + if not self.is_running: + return if any(x in updated for x in ["server", "listen_host", "listen_port"]): asyncio.create_task(self.refresh_server()) diff --git a/mitmproxy/platform/windows.py b/mitmproxy/platform/windows.py index b447a75179..270731dace 100644 --- a/mitmproxy/platform/windows.py +++ b/mitmproxy/platform/windows.py @@ -32,6 +32,8 @@ def winerror(self) -> int: def read(rfile: io.BufferedReader) -> typing.Any: x = rfile.readline().strip() + if not x: + return None return json.loads(x) @@ -89,11 +91,15 @@ def handle(self): proxifier: TransparentProxy = self.server.proxifier try: pid: int = read(self.rfile) + if pid is None: + return with proxifier.exempt(pid): while True: - client = tuple(read(self.rfile)) + c = read(self.rfile) + if c is None: + return try: - server = proxifier.client_server_map[client] + server = proxifier.client_server_map[tuple(c)] except KeyError: server = None write(server, self.wfile) @@ -575,7 +581,7 @@ def redirect(**options): proxy = TransparentProxy(**options) proxy.start() print(f" * Redirection active.") - print(f" Filter: {proxy.request_filter}") + print(f" Filter: {proxy.filter}") try: while True: time.sleep(1) From 39fa242e25ad6d879e7021db188d0d32f4d77115 Mon Sep 17 00:00:00 2001 From: shindexro Date: Wed, 17 Nov 2021 16:53:43 +0000 Subject: [PATCH 013/285] fix crash when invoking `replay.server.count` from console (#4905) fix #4902 --- mitmproxy/tools/console/commandexecutor.py | 2 +- mitmproxy/tools/console/grideditor/editors.py | 6 +++--- test/mitmproxy/tools/console/test_integration.py | 5 +++++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/mitmproxy/tools/console/commandexecutor.py b/mitmproxy/tools/console/commandexecutor.py index 1c6d5aa696..69dbb9a3f3 100644 --- a/mitmproxy/tools/console/commandexecutor.py +++ b/mitmproxy/tools/console/commandexecutor.py @@ -19,7 +19,7 @@ def __call__(self, cmd): except exceptions.CommandError as e: ctx.log.error(str(e)) else: - if ret: + if ret is not None: if type(ret) == typing.Sequence[flow.Flow]: signals.status_message.send( message="Command returned %s flows" % len(ret) diff --git a/mitmproxy/tools/console/grideditor/editors.py b/mitmproxy/tools/console/grideditor/editors.py index cf3aa05a67..7f65139d58 100644 --- a/mitmproxy/tools/console/grideditor/editors.py +++ b/mitmproxy/tools/console/grideditor/editors.py @@ -213,11 +213,11 @@ def __init__( vals: typing.Union[ typing.List[typing.List[typing.Any]], typing.List[typing.Any], - str, + typing.Any, ]) -> None: - if vals: + if vals is not None: # Whatever vals is, make it a list of rows containing lists of column values. - if isinstance(vals, str): + if not isinstance(vals, list): vals = [vals] if not isinstance(vals[0], list): vals = [[i] for i in vals] diff --git a/test/mitmproxy/tools/console/test_integration.py b/test/mitmproxy/tools/console/test_integration.py index ed80bad2fd..15c1568238 100644 --- a/test/mitmproxy/tools/console/test_integration.py +++ b/test/mitmproxy/tools/console/test_integration.py @@ -54,3 +54,8 @@ def test_options_home_end(console): @pytest.mark.asyncio def test_keybindings_home_end(console): console.type("K") + + +@pytest.mark.asyncio +def test_replay_count(console): + console.type(":replay.server.count") From 888ce66f902add53d7fcbb2df44ed010229098fd Mon Sep 17 00:00:00 2001 From: shindexro Date: Thu, 18 Nov 2021 07:26:05 +0000 Subject: [PATCH 014/285] Correct flow-detail documentation (#4909) * Fix #4902 * Update type signature * Switch to None check * Fix spacing * Skip URL shortening when flow-detail is 0 * Sync docs with implementation * Update URL shortening test --- mitmproxy/addons/dumper.py | 10 +++++----- test/mitmproxy/addons/test_dumper.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mitmproxy/addons/dumper.py b/mitmproxy/addons/dumper.py index 233bf8596c..aeb4d2054a 100644 --- a/mitmproxy/addons/dumper.py +++ b/mitmproxy/addons/dumper.py @@ -38,10 +38,10 @@ def load(self, loader): loader.add_option( "flow_detail", int, 1, """ - The display detail level for flows in mitmdump: 0 (almost quiet) to 3 (very verbose). - 0: shortened request URL, response status code, WebSocket and TCP message notifications. - 1: full request URL with response status code - 2: 1 + HTTP headers + The display detail level for flows in mitmdump: 0 (quiet) to 4 (very verbose). + 0: no output + 1: shortened request URL with response status code + 2: full request URL with response status code and HTTP headers 3: 2 + truncated response content, content of WebSocket and TCP messages 4: 3 + nothing is truncated """ @@ -156,7 +156,7 @@ def _echo_request_line(self, flow: http.HTTPFlow) -> None: else: url = flow.request.url - if ctx.options.flow_detail <= 1: + if ctx.options.flow_detail == 1: # We need to truncate before applying styles, so we just focus on the URL. terminal_width_limit = max(shutil.get_terminal_size()[0] - 25, 50) if len(url) > terminal_width_limit: diff --git a/test/mitmproxy/addons/test_dumper.py b/test/mitmproxy/addons/test_dumper.py index 5e9ed7639e..a63e878a87 100644 --- a/test/mitmproxy/addons/test_dumper.py +++ b/test/mitmproxy/addons/test_dumper.py @@ -158,7 +158,7 @@ def test_echo_request_line(): assert "nonstandard" in sio.getvalue() sio.truncate(0) - ctx.configure(d, flow_detail=0, showhost=True) + ctx.configure(d, flow_detail=1, showhost=True) f = tflow.tflow(resp=True) terminalWidth = max(shutil.get_terminal_size()[0] - 25, 50) f.request.url = "http://address:22/" + ("x" * terminalWidth) + "textToBeTruncated" From 0904cc3f5f0f95f6b9c3a6829e02f08ebb5bf9e7 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 18 Nov 2021 08:45:28 +0100 Subject: [PATCH 015/285] console: add error message for older Windows versions (#4911) --- mitmproxy/contrib/urwid/raw_display.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mitmproxy/contrib/urwid/raw_display.py b/mitmproxy/contrib/urwid/raw_display.py index ee6d30bdf3..337c63df62 100644 --- a/mitmproxy/contrib/urwid/raw_display.py +++ b/mitmproxy/contrib/urwid/raw_display.py @@ -260,7 +260,9 @@ def _start(self, alternate_buffer=True): ) ok = win32.SetConsoleMode(hOut, dwOutMode) - assert ok + if not ok: + raise RuntimeError("Error enabling virtual terminal processing, " + "mitmproxy's console interface requires Windows 10 Build 10586 or above.") ok = win32.SetConsoleMode(hIn, dwInMode) assert ok else: From 7be646f44a14eb167a20094a526ceab1227df8fb Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 18 Nov 2021 15:39:14 +0100 Subject: [PATCH 016/285] add note that spoof_source_address is unavailable, refs #4914 --- docs/src/content/howto-transparent.md | 5 +++++ mitmproxy/utils/arg_check.py | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/src/content/howto-transparent.md b/docs/src/content/howto-transparent.md index b143f1a475..205ddf68a7 100644 --- a/docs/src/content/howto-transparent.md +++ b/docs/src/content/howto-transparent.md @@ -278,6 +278,11 @@ sudo -u nobody mitmproxy --mode transparent --showhost ## "Full" transparent mode on Linux +{{% note %}} +This feature is currently unavailable in mitmproxy 7 and above +(#4914). +{{% /note %}} + By default mitmproxy will use its own local IP address for its server-side connections. In case this isn't desired, the --spoof-source-address argument can be used to use the client's IP address for server-side connections. The diff --git a/mitmproxy/utils/arg_check.py b/mitmproxy/utils/arg_check.py index 4bb1f18122..e6cb61cc3f 100644 --- a/mitmproxy/utils/arg_check.py +++ b/mitmproxy/utils/arg_check.py @@ -16,7 +16,6 @@ --no-http2-priority --no-websocket --websocket ---spoof-source-address --upstream-bind-address --ciphers-client --ciphers-server From 9a469806eb6a202f5b35c011368fd240a16f5397 Mon Sep 17 00:00:00 2001 From: shindexro Date: Fri, 19 Nov 2021 12:04:20 +0000 Subject: [PATCH 017/285] quote argument of view.flows.resolve (#4910) * Fix #4902 * Update type signature * Switch to None check * Fix spacing * Quote view.flows.resolve argument * Switch to call_strings --- mitmproxy/types.py | 4 ++-- test/mitmproxy/test_types.py | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/mitmproxy/types.py b/mitmproxy/types.py index 01d7ac47f8..b4b17d80ea 100644 --- a/mitmproxy/types.py +++ b/mitmproxy/types.py @@ -369,7 +369,7 @@ class _FlowType(_BaseFlowType): def parse(self, manager: "CommandManager", t: type, s: str) -> flow.Flow: try: - flows = manager.execute("view.flows.resolve %s" % (s)) + flows = manager.call_strings("view.flows.resolve", [s]) except exceptions.CommandError as e: raise exceptions.TypeError(str(e)) from e if len(flows) != 1: @@ -388,7 +388,7 @@ class _FlowsType(_BaseFlowType): def parse(self, manager: "CommandManager", t: type, s: str) -> typing.Sequence[flow.Flow]: try: - return manager.execute("view.flows.resolve %s" % (s)) + return manager.call_strings("view.flows.resolve", [s]) except exceptions.CommandError as e: raise exceptions.TypeError(str(e)) from e diff --git a/test/mitmproxy/test_types.py b/test/mitmproxy/test_types.py index e0f56995f9..7b7d203cde 100644 --- a/test/mitmproxy/test_types.py +++ b/test/mitmproxy/test_types.py @@ -183,7 +183,10 @@ class DummyConsole: def resolve(self, spec: str) -> typing.Sequence[flow.Flow]: if spec == "err": raise mitmproxy.exceptions.CommandError() - n = int(spec) + try: + n = int(spec) + except ValueError: + n = 1 return [tflow.tflow(resp=True)] * n @command.command("cut") @@ -201,6 +204,7 @@ def test_flow(): b = mitmproxy.types._FlowType() assert len(b.completion(tctx.master.commands, flow.Flow, "")) == len(b.valid_prefixes) assert b.parse(tctx.master.commands, flow.Flow, "1") + assert b.parse(tctx.master.commands, flow.Flow, "has space") assert b.is_valid(tctx.master.commands, flow.Flow, tflow.tflow()) is True assert b.is_valid(tctx.master.commands, flow.Flow, "xx") is False with pytest.raises(mitmproxy.exceptions.TypeError): @@ -224,6 +228,7 @@ def test_flows(): assert len(b.parse(tctx.master.commands, typing.Sequence[flow.Flow], "0")) == 0 assert len(b.parse(tctx.master.commands, typing.Sequence[flow.Flow], "1")) == 1 assert len(b.parse(tctx.master.commands, typing.Sequence[flow.Flow], "2")) == 2 + assert len(b.parse(tctx.master.commands, typing.Sequence[flow.Flow], "has space")) == 1 with pytest.raises(mitmproxy.exceptions.TypeError): b.parse(tctx.master.commands, typing.Sequence[flow.Flow], "err") From 3847842f731c8e1c5a5990ef5c73897e4ab91028 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Sun, 21 Nov 2021 09:14:47 -0500 Subject: [PATCH 018/285] add runtime dependency on setuptools (#4919) resolves #4918 Co-authored-by: Maximilian Hils --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 59ef56389b..2ab7a6fea5 100644 --- a/setup.py +++ b/setup.py @@ -86,6 +86,8 @@ "pyparsing>=2.4.2,<2.5", "pyperclip>=1.6.0,<1.9", "ruamel.yaml>=0.16,<0.17.17", + # Kaitai parsers depend on setuptools, remove once https://github.com/kaitai-io/kaitai_struct_python_runtime/issues/62 is fixed + "setuptools", "sortedcontainers>=2.3,<2.5", "tornado>=6.1,<7", "urwid>=2.1.1,<2.2", From 1c93a9369683b1d6f978d4cbee7e528f58ca8b09 Mon Sep 17 00:00:00 2001 From: Marius Date: Sun, 21 Nov 2021 08:47:09 -0600 Subject: [PATCH 019/285] Add font types to asset filter (~a) (#4928) * Add font types to asset filter (~a) * Add PR number to changelog * remove flash mention * restore asset test Co-authored-by: Maximilian Hils --- CHANGELOG.md | 1 + mitmproxy/flowfilter.py | 9 ++++++--- web/src/js/filt/filt.peg | 4 +++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8db2dd7712..6c52595fc8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ * Base container image bumped to Debian 11 Bullseye (@Kriechi) * Upstream replays don't do CONNECT on plaintext HTTP requests (#4876, @HoffmannP) * Remove workarounds for old pyOpenSSL versions (#4831, @KarlParkinson) +* Add fonts to asset filter (~a) (#4928, @elespike) ## 28 September 2021: mitmproxy 7.0.4 diff --git a/mitmproxy/flowfilter.py b/mitmproxy/flowfilter.py index 29ba0af841..da3f9d0d8f 100644 --- a/mitmproxy/flowfilter.py +++ b/mitmproxy/flowfilter.py @@ -15,7 +15,8 @@ application/javascript text/css image/* - application/x-shockwave-flash + font/* + application/font-* ~h rex Header line in either request or response ~hq rex Header in request ~hs rex Header in response @@ -167,13 +168,15 @@ def _check_content_type(rex, message): class FAsset(_Action): code = "a" - help = "Match asset in response: CSS, JavaScript, images." + help = "Match asset in response: CSS, JavaScript, images, fonts." ASSET_TYPES = [re.compile(x) for x in [ b"text/javascript", b"application/x-javascript", b"application/javascript", b"text/css", - b"image/.*" + b"image/.*", + b"font/.*", + b"application/font-.*", ]] @only(http.HTTPFlow) diff --git a/web/src/js/filt/filt.peg b/web/src/js/filt/filt.peg index bca8bdddf0..360324562b 100644 --- a/web/src/js/filt/filt.peg +++ b/web/src/js/filt/filt.peg @@ -45,7 +45,9 @@ var ASSET_TYPES = [ new RegExp("application/x-javascript"), new RegExp("application/javascript"), new RegExp("text/css"), - new RegExp("image/.*") + new RegExp("image/.*"), + new RegExp("font/.*"), + new RegExp("application/font.*"), ]; function assetFilter(flow) { if (flow.response) { From 6398dc308a85fb3e42aaeb09ae311c05afe92daa Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 22 Nov 2021 07:41:33 +0100 Subject: [PATCH 020/285] WebSockets: don't assert that server is connected upon initialization (#4932) there may already have been a disconnect, which is still in the processing queue. fix #4931 --- mitmproxy/proxy/layers/websocket.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mitmproxy/proxy/layers/websocket.py b/mitmproxy/proxy/layers/websocket.py index b616eb9408..16cbf657fe 100644 --- a/mitmproxy/proxy/layers/websocket.py +++ b/mitmproxy/proxy/layers/websocket.py @@ -86,7 +86,6 @@ class WebsocketLayer(layer.Layer): def __init__(self, context: Context, flow: http.HTTPFlow): super().__init__(context) self.flow = flow - assert context.server.connected @expect(events.Start) def start(self, _) -> layer.CommandGenerator[None]: From 3cb87f5a2f00790535e9ffa3c7ccf3566291a055 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 22 Nov 2021 10:10:14 +0100 Subject: [PATCH 021/285] split `tls_handshake` hook into client/server and success/fail variants --- CHANGELOG.md | 6 +--- docs/scripts/api-events.py | 5 +++- examples/contrib/tls_passthrough.py | 17 +++++------ mitmproxy/proxy/layers/tls.py | 40 +++++++++++++++++++++---- test/mitmproxy/proxy/layers/test_tls.py | 26 +++++++++------- 5 files changed, 62 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7c07c3a90..cee75fc2df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,12 +4,8 @@ * Support proxy authentication for SOCKS v5 mode (@starplanet) * Make it possible to ignore connections in the tls_clienthello event hook (@mhils) -* Add `tls_handshake` event hook to record negotiation success/failure (@mhils) +* Add `tls_established/failed_client/server` event hooks to record negotiation success/failure (@mhils) * fix some responses not being decoded properly if the encoding was uppercase #4735 (@Mattwmaster58) -* Expose TLS 1.0 as possible minimum version on older pyOpenSSL releases (@mhils) -* Improve error message on TLS version mismatch. (@mhils) -* Windows: Switch to Python's default asyncio event loop, which increases the number of sockets - that can be processed simultaneously (@mhils) * Trigger event hooks for flows with semantically invalid requests, for example invalid content-length headers (@mhils) * Improve error message on TLS version mismatch (@mhils) * Windows: Switch to Python's default asyncio event loop, which increases the number of sockets diff --git a/docs/scripts/api-events.py b/docs/scripts/api-events.py index 462c025310..eca751a79f 100644 --- a/docs/scripts/api-events.py +++ b/docs/scripts/api-events.py @@ -124,7 +124,10 @@ def category(name: str, desc: str, hooks: List[Type[hooks.Hook]]) -> None: tls.TlsClienthelloHook, tls.TlsStartClientHook, tls.TlsStartServerHook, - tls.TlsHandshakeHook, + tls.TlsEstablishedClientHook, + tls.TlsEstablishedServerHook, + tls.TlsFailedClientHook, + tls.TlsFailedServerHook, ] ) diff --git a/examples/contrib/tls_passthrough.py b/examples/contrib/tls_passthrough.py index 182f26289d..811d56abfe 100644 --- a/examples/contrib/tls_passthrough.py +++ b/examples/contrib/tls_passthrough.py @@ -94,16 +94,15 @@ def tls_clienthello(self, data: tls.ClientHelloData): data.ignore_connection = True self.strategy.record_skipped(server_address) - def tls_handshake(self, data: tls.TlsData): - if isinstance(data.conn, connection.Server): - return # we are only interested in failing client connections here. + def tls_established_client(self, data: tls.TlsData): server_address = data.context.server.peername - if data.conn.error is None: - ctx.log(f"TLS handshake successful: {human.format_address(server_address)}") - self.strategy.record_success(server_address) - else: - ctx.log(f"TLS handshake failed: {human.format_address(server_address)}") - self.strategy.record_failure(server_address) + ctx.log(f"TLS handshake successful: {human.format_address(server_address)}") + self.strategy.record_success(server_address) + + def tls_failed_client(self, data: tls.TlsData): + server_address = data.context.server.peername + ctx.log(f"TLS handshake failed: {human.format_address(server_address)}") + self.strategy.record_failure(server_address) addons = [MaybeTls()] diff --git a/mitmproxy/proxy/layers/tls.py b/mitmproxy/proxy/layers/tls.py index 99eb146813..924a0fff7b 100644 --- a/mitmproxy/proxy/layers/tls.py +++ b/mitmproxy/proxy/layers/tls.py @@ -133,11 +133,33 @@ class TlsStartServerHook(StartHook): @dataclass -class TlsHandshakeHook(StartHook): +class TlsEstablishedClientHook(StartHook): """ - A TLS handshake has been completed. + The TLS handshake with the client has been completed successfully. + """ + data: TlsData + + +@dataclass +class TlsEstablishedServerHook(StartHook): + """ + The TLS handshake with the server has been completed successfully. + """ + data: TlsData + + +@dataclass +class TlsFailedClientHook(StartHook): + """ + The TLS handshake with the client has failed. + """ + data: TlsData + - If `data.conn.error` is `None`, negotiation was successful. +@dataclass +class TlsFailedServerHook(StartHook): + """ + The TLS handshake with the server has failed. """ data: TlsData @@ -162,7 +184,7 @@ def start_tls(self) -> layer.CommandGenerator[None]: assert not self.tls tls_start = TlsData(self.conn, self.context) - if tls_start.conn == tls_start.context.client: + if self.conn == self.context.client: yield TlsStartClientHook(tls_start) else: yield TlsStartServerHook(tls_start) @@ -234,13 +256,19 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[Tuple[bo self.conn.tls_version = self.tls.get_protocol_version_name() if self.debug: yield commands.Log(f"{self.debug}[tls] tls established: {self.conn}", "debug") - yield TlsHandshakeHook(TlsData(self.conn, self.context, self.tls)) + if self.conn == self.context.client: + yield TlsEstablishedClientHook(TlsData(self.conn, self.context, self.tls)) + else: + yield TlsEstablishedServerHook(TlsData(self.conn, self.context, self.tls)) yield from self.receive_data(b"") return True, None def on_handshake_error(self, err: str) -> layer.CommandGenerator[None]: self.conn.error = err - yield TlsHandshakeHook(TlsData(self.conn, self.context, self.tls)) + if self.conn == self.context.client: + yield TlsFailedClientHook(TlsData(self.conn, self.context, self.tls)) + else: + yield TlsFailedServerHook(TlsData(self.conn, self.context, self.tls)) yield from super().on_handshake_error(err) def receive_data(self, data: bytes) -> layer.CommandGenerator[None]: diff --git a/test/mitmproxy/proxy/layers/test_tls.py b/test/mitmproxy/proxy/layers/test_tls.py index e20537c34c..651e2c87ef 100644 --- a/test/mitmproxy/proxy/layers/test_tls.py +++ b/test/mitmproxy/proxy/layers/test_tls.py @@ -158,10 +158,14 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: def finish_handshake(playbook: tutils.Playbook, conn: connection.Connection, tssl: SSLTest): data = tutils.Placeholder(bytes) tls_hook_data = tutils.Placeholder(TlsData) + if isinstance(conn, connection.Client): + established_hook = tls.TlsEstablishedClientHook(tls_hook_data) + else: + established_hook = tls.TlsEstablishedServerHook(tls_hook_data) assert ( playbook >> events.DataReceived(conn, tssl.bio_read()) - << tls.TlsHandshakeHook(tls_hook_data) + << established_hook >> tutils.reply() << commands.SendData(conn, data) ) @@ -334,7 +338,7 @@ def test_untrusted_cert(self, tctx): playbook >> events.DataReceived(tctx.server, tssl.bio_read()) << commands.Log("Server TLS handshake failed. Certificate verify failed: Hostname mismatch", "warn") - << tls.TlsHandshakeHook(tls_hook_data) + << tls.TlsFailedServerHook(tls_hook_data) >> tutils.reply() << commands.CloseConnection(tctx.server) << commands.SendData(tctx.client, @@ -358,7 +362,7 @@ def test_remote_speaks_no_tls(self, tctx): << commands.SendData(tctx.server, data) >> events.DataReceived(tctx.server, b"HTTP/1.1 404 Not Found\r\n") << commands.Log("Server TLS handshake failed. The remote server does not speak TLS.", "warn") - << tls.TlsHandshakeHook(tls_hook_data) + << tls.TlsFailedServerHook(tls_hook_data) >> tutils.reply() << commands.CloseConnection(tctx.server) ) @@ -395,7 +399,7 @@ def test_unsupported_protocol(self, tctx: context.Context): >> events.DataReceived(tctx.server, tssl.bio_read()) << commands.Log("Server TLS handshake failed. The remote server and mitmproxy cannot agree on a TLS version" " to use. You may need to adjust mitmproxy's tls_version_server_min option.", "warn") - << tls.TlsHandshakeHook(tls_hook_data) + << tls.TlsFailedServerHook(tls_hook_data) >> tutils.reply() << commands.CloseConnection(tctx.server) ) @@ -506,7 +510,7 @@ def require_server_conn(client_hello: ClientHelloData) -> None: assert ( playbook >> events.DataReceived(tctx.server, tssl_server.bio_read()) - << tls.TlsHandshakeHook(tutils.Placeholder()) + << tls.TlsEstablishedServerHook(tutils.Placeholder()) >> tutils.reply() << commands.SendData(tctx.server, data) << tls.TlsStartClientHook(tutils.Placeholder()) @@ -579,7 +583,7 @@ def test_cannot_parse_clienthello(self, tctx: context.Context): playbook >> events.DataReceived(tctx.client, invalid) << commands.Log(f"Client TLS handshake failed. Cannot parse ClientHello: {invalid.hex()}", level="warn") - << tls.TlsHandshakeHook(tls_hook_data) + << tls.TlsFailedClientHook(tls_hook_data) >> tutils.reply() << commands.CloseConnection(tctx.client) ) @@ -620,7 +624,7 @@ def test_mitmproxy_ca_is_untrusted(self, tctx: context.Context): >> events.DataReceived(tctx.client, tssl_client.bio_read()) << commands.Log("Client TLS handshake failed. The client does not trust the proxy's certificate " "for wrong.host.mitmproxy.org (sslv3 alert bad certificate)", "warn") - << tls.TlsHandshakeHook(tls_hook_data) + << tls.TlsFailedClientHook(tls_hook_data) >> tutils.reply() << commands.CloseConnection(tctx.client) >> events.ConnectionClosed(tctx.client) @@ -647,7 +651,7 @@ def test_immediate_disconnect(self, tctx: context.Context, close_at): >> tutils.reply(to=-2) << tls.TlsStartClientHook(tutils.Placeholder()) >> reply_tls_start_client() - << tls.TlsHandshakeHook(tls_hook_data) + << tls.TlsFailedClientHook(tls_hook_data) >> tutils.reply() << commands.CloseConnection(tctx.client) ) @@ -662,7 +666,7 @@ def test_immediate_disconnect(self, tctx: context.Context, close_at): playbook >> events.ConnectionClosed(tctx.client) >> reply_tls_start_client(to=-2) - << tls.TlsHandshakeHook(tls_hook_data) + << tls.TlsFailedClientHook(tls_hook_data) >> tutils.reply() << commands.CloseConnection(tctx.client) ) @@ -677,7 +681,7 @@ def test_immediate_disconnect(self, tctx: context.Context, close_at): << commands.Log("Client TLS handshake failed. The client disconnected during the handshake. " "If this happens consistently for wrong.host.mitmproxy.org, this may indicate that the " "client does not trust the proxy's certificate.", "info") - << tls.TlsHandshakeHook(tls_hook_data) + << tls.TlsFailedClientHook(tls_hook_data) >> tutils.reply() << commands.CloseConnection(tctx.client) ) @@ -698,7 +702,7 @@ def test_unsupported_protocol(self, tctx: context.Context): >> reply_tls_start_client() << commands.Log("Client TLS handshake failed. Client and mitmproxy cannot agree on a TLS version to " "use. You may need to adjust mitmproxy's tls_version_client_min option.", "warn") - << tls.TlsHandshakeHook(tls_hook_data) + << tls.TlsFailedClientHook(tls_hook_data) >> tutils.reply() << commands.CloseConnection(tctx.client) ) From eea24bc22dccd0f8bd5b6e6e7d3de2369d725704 Mon Sep 17 00:00:00 2001 From: "requires.io" Date: Mon, 22 Nov 2021 10:50:47 +0100 Subject: [PATCH 022/285] [requires.io] dependency update --- tox.ini | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tox.ini b/tox.ini index cbd69e163c..d76276263a 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,7 @@ commands = [testenv:flake8] deps = - flake8>=3.8.4,<4 + flake8>=4.0,<4.1 flake8-tidy-imports>=4.2.0,<5 commands = flake8 --jobs 8 mitmproxy examples test release {posargs} @@ -30,11 +30,11 @@ commands = [testenv:mypy] deps = mypy==0.910 - types-certifi==2020.4.0 - types-Flask==1.1.3 - types-Werkzeug==1.0.5 - types-requests==2.25.9 - types-cryptography==3.3.5 + types-certifi==2021.10.8.0 + types-Flask==1.1.5 + types-Werkzeug==1.0.7 + types-requests==2.26.0 + types-cryptography==3.3.8 types-pyOpenSSL==21.0.0 commands = From dae098cd33194c9a0d42cde2ffdf2ceba239943b Mon Sep 17 00:00:00 2001 From: "requires.io" Date: Mon, 22 Nov 2021 10:50:48 +0100 Subject: [PATCH 023/285] [requires.io] dependency update --- setup.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index 2ab7a6fea5..3391aead7b 100644 --- a/setup.py +++ b/setup.py @@ -72,7 +72,7 @@ "Brotli>=1.0,<1.1", "certifi>=2019.9.11", # no semver here - this should always be on the last release! "click>=7.0,<8.1", - "cryptography>=3.3,<3.5", + "cryptography>=36.0,<36.1", "flask>=1.1.1,<2.1", "h11>=0.11,<0.13", "h2>=4.1,<5", @@ -81,11 +81,11 @@ "ldap3>=2.8,<2.10", "msgpack>=1.0.0, <1.1.0", "passlib>=1.6.5, <1.8", - "protobuf>=3.14,<3.19", + "protobuf>=3.19,<3.20", "pyOpenSSL>=21.0,<21.1", - "pyparsing>=2.4.2,<2.5", + "pyparsing>=3.0,<3.1", "pyperclip>=1.6.0,<1.9", - "ruamel.yaml>=0.16,<0.17.17", + "ruamel.yaml>=0.17,<0.18", # Kaitai parsers depend on setuptools, remove once https://github.com/kaitai-io/kaitai_struct_python_runtime/issues/62 is fixed "setuptools", "sortedcontainers>=2.3,<2.5", @@ -93,7 +93,7 @@ "urwid>=2.1.1,<2.2", "wsproto>=1.0,<1.1", "publicsuffix2>=2.20190812,<3", - "zstandard>=0.11,<0.16", + "zstandard>=0.16,<0.17", ], extras_require={ ':sys_platform == "win32"': [ @@ -103,16 +103,16 @@ "hypothesis>=5.8,<7", "parver>=0.1,<2.0", "pdoc>=4.0.0", - "pyinstaller==4.5.1", + "pyinstaller==4.7", "pytest-asyncio>=0.10.0,<0.16,!=0.14", - "pytest-cov>=2.7.1,<3", - "pytest-timeout>=1.3.3,<2", + "pytest-cov>=3.0,<3.1", + "pytest-timeout>=2.0,<2.1", "pytest-xdist>=2.1.0,<3", "pytest>=6.1.0,<7", "requests>=2.9.1,<3", "tox>=3.5,<4", "wheel>=0.36.2,<0.38", - "coverage==5.5", # workaround issue with import errors introduced in 5.6b1/6.0 + "coverage==6.1.2", # workaround issue with import errors introduced in 5.6b1/6.0 ], } ) From 9249c0ddd37a55657e3714bff0bbef5bba464631 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 22 Nov 2021 11:09:04 +0100 Subject: [PATCH 024/285] adjust lower version bounds --- setup.py | 15 +++++++-------- tox.ini | 4 ++-- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/setup.py b/setup.py index 3391aead7b..b41c456bcf 100644 --- a/setup.py +++ b/setup.py @@ -72,7 +72,7 @@ "Brotli>=1.0,<1.1", "certifi>=2019.9.11", # no semver here - this should always be on the last release! "click>=7.0,<8.1", - "cryptography>=36.0,<36.1", + "cryptography>=3.3,<37,!=35", "flask>=1.1.1,<2.1", "h11>=0.11,<0.13", "h2>=4.1,<5", @@ -81,11 +81,11 @@ "ldap3>=2.8,<2.10", "msgpack>=1.0.0, <1.1.0", "passlib>=1.6.5, <1.8", - "protobuf>=3.19,<3.20", + "protobuf>=3.14,<3.20", "pyOpenSSL>=21.0,<21.1", - "pyparsing>=3.0,<3.1", + "pyparsing>=2.4.2,<3.1", "pyperclip>=1.6.0,<1.9", - "ruamel.yaml>=0.17,<0.18", + "ruamel.yaml>=0.16,<0.18", # Kaitai parsers depend on setuptools, remove once https://github.com/kaitai-io/kaitai_struct_python_runtime/issues/62 is fixed "setuptools", "sortedcontainers>=2.3,<2.5", @@ -93,7 +93,7 @@ "urwid>=2.1.1,<2.2", "wsproto>=1.0,<1.1", "publicsuffix2>=2.20190812,<3", - "zstandard>=0.16,<0.17", + "zstandard>=0.11,<0.17", ], extras_require={ ':sys_platform == "win32"': [ @@ -105,14 +105,13 @@ "pdoc>=4.0.0", "pyinstaller==4.7", "pytest-asyncio>=0.10.0,<0.16,!=0.14", - "pytest-cov>=3.0,<3.1", - "pytest-timeout>=2.0,<2.1", + "pytest-cov>=2.7.1,<3.1", + "pytest-timeout>=1.3.3,<2.1", "pytest-xdist>=2.1.0,<3", "pytest>=6.1.0,<7", "requests>=2.9.1,<3", "tox>=3.5,<4", "wheel>=0.36.2,<0.38", - "coverage==6.1.2", # workaround issue with import errors introduced in 5.6b1/6.0 ], } ) diff --git a/tox.ini b/tox.ini index d76276263a..0a28c83167 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,7 @@ commands = [testenv:flake8] deps = - flake8>=4.0,<4.1 + flake8>=3.8.4,<4.1 flake8-tidy-imports>=4.2.0,<5 commands = flake8 --jobs 8 mitmproxy examples test release {posargs} @@ -39,7 +39,7 @@ deps = commands = mypy {posargs} - + [testenv:individual_coverage] commands = python ./test/individual_coverage.py {posargs} From 95089486ca1be80b408f32d131b62a5ec7d75ce6 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 22 Nov 2021 11:14:15 +0100 Subject: [PATCH 025/285] cryptography now accepts some more invalid attrs - remove dead code paths --- mitmproxy/addons/tlsconfig.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/mitmproxy/addons/tlsconfig.py b/mitmproxy/addons/tlsconfig.py index 548a95a879..416532bc9f 100644 --- a/mitmproxy/addons/tlsconfig.py +++ b/mitmproxy/addons/tlsconfig.py @@ -284,18 +284,11 @@ def get_cert(self, conn_context: context.Context) -> certs.CertStoreEntry: # Use upstream certificate if available. if ctx.options.upstream_cert and conn_context.server.certificate_list: upstream_cert = conn_context.server.certificate_list[0] - try: - # a bit clunky: access to .cn can fail, see https://github.com/mitmproxy/mitmproxy/issues/4713 - if upstream_cert.cn: - altnames.append(upstream_cert.cn) - except ValueError: - pass + if upstream_cert.cn: + altnames.append(upstream_cert.cn) altnames.extend(upstream_cert.altnames) - try: - if upstream_cert.organization: - organization = upstream_cert.organization - except ValueError: - pass + if upstream_cert.organization: + organization = upstream_cert.organization # Add SNI. If not available, try the server address as well. if conn_context.client.sni: From 68bce90754a546d15ea9a5fa75ff324da3a18086 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 22 Nov 2021 12:52:37 +0100 Subject: [PATCH 026/285] require cryptography 36+ 3.3+ should mostly work fine, but our tests cover v36-specific behavior --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b41c456bcf..934affc8e3 100644 --- a/setup.py +++ b/setup.py @@ -72,7 +72,7 @@ "Brotli>=1.0,<1.1", "certifi>=2019.9.11", # no semver here - this should always be on the last release! "click>=7.0,<8.1", - "cryptography>=3.3,<37,!=35", + "cryptography>=36,<37", "flask>=1.1.1,<2.1", "h11>=0.11,<0.13", "h2>=4.1,<5", From 9803c90341fd741893dd14081834cbfdcce4dceb Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 22 Nov 2021 12:54:02 +0100 Subject: [PATCH 027/285] make it possible to disable exit on `SIGUSR1` --- mitmproxy/utils/debug.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mitmproxy/utils/debug.py b/mitmproxy/utils/debug.py index 986372efa2..58e1848a21 100644 --- a/mitmproxy/utils/debug.py +++ b/mitmproxy/utils/debug.py @@ -99,7 +99,7 @@ def dump_info(signal=None, frame=None, file=sys.stdout, testing=False): # pragm print("****************************************************") - if not testing: + if not testing and not os.getenv("MITMPROXY_DEBUG_STAY_ALIVE"): # pragma: no cover sys.exit(1) @@ -117,7 +117,7 @@ def dump_stacks(signal=None, frame=None, file=sys.stdout, testing=False): if line: code.append(" %s" % (line.strip())) print("\n".join(code), file=file) - if not testing: # pragma: no cover + if not testing and not os.getenv("MITMPROXY_DEBUG_STAY_ALIVE"): # pragma: no cover sys.exit(1) From 2dd845ed95db54be2a9548e54f3fcdc6c82d472a Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 22 Nov 2021 12:55:17 +0100 Subject: [PATCH 028/285] catch malformed cert warning in tests we may need to catch this properly in `get_cert` at some point, let's see if this ever turns out to be an issue. --- test/mitmproxy/addons/test_tlsconfig.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/mitmproxy/addons/test_tlsconfig.py b/test/mitmproxy/addons/test_tlsconfig.py index 115d92c0ce..2f203713dc 100644 --- a/test/mitmproxy/addons/test_tlsconfig.py +++ b/test/mitmproxy/addons/test_tlsconfig.py @@ -86,7 +86,8 @@ def test_get_cert(self, tdata): with open(tdata.path("mitmproxy/data/invalid-subject.pem"), "rb") as f: ctx.server.certificate_list = [certs.Cert.from_pem(f.read())] - assert ta.get_cert(ctx) # does not raise + with pytest.warns(UserWarning, match="Country names should be two characters"): + assert ta.get_cert(ctx) # does not raise def test_tls_clienthello(self): # only really testing for coverage here, there's no point in mirroring the individual conditions From 27e69651f6c5d6cbd4a8dbe549d809bc68cc3446 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 23 Nov 2021 21:12:32 +0100 Subject: [PATCH 029/285] [requires.io] dependency update (#4946) Co-authored-by: requires.io --- tox.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 0a28c83167..27ec66667e 100644 --- a/tox.ini +++ b/tox.ini @@ -32,10 +32,10 @@ deps = mypy==0.910 types-certifi==2021.10.8.0 types-Flask==1.1.5 - types-Werkzeug==1.0.7 + types-Werkzeug==1.0.8 types-requests==2.26.0 - types-cryptography==3.3.8 - types-pyOpenSSL==21.0.0 + types-cryptography==3.3.9 + types-pyOpenSSL==21.0.1 commands = mypy {posargs} From b1c1cd3f2f4b32c2569597573181f11352d71c5d Mon Sep 17 00:00:00 2001 From: Felix Ingram Date: Wed, 24 Nov 2021 15:49:47 +0000 Subject: [PATCH 030/285] Fixes AttributeError in transparent mode (#4951) If we haven't connected to the upstream host in transparent mode, then the call to `write` on line 74 will raise an `AttributeError` as we haven't initialised `self.wfile`. If we add `AttributeError` to the `except` line, then we'll catch the exception and call `_connect`, which will in turn initialise `wfile`. --- mitmproxy/platform/windows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitmproxy/platform/windows.py b/mitmproxy/platform/windows.py index 270731dace..966873e94a 100644 --- a/mitmproxy/platform/windows.py +++ b/mitmproxy/platform/windows.py @@ -76,7 +76,7 @@ def original_addr(self, csock: socket.socket): if addr is None: raise RuntimeError("Cannot resolve original destination.") return tuple(addr) - except (EOFError, OSError): + except (EOFError, OSError, AttributeError): self._connect() return self.original_addr(csock) From b1d0887db4aaf3e0f66a6058f03e4ca657f663f9 Mon Sep 17 00:00:00 2001 From: Brad Dixon Date: Fri, 26 Nov 2021 17:14:06 -0500 Subject: [PATCH 031/285] Add deepcopy to allow view.flows.resolve (issue #4916) (#4952) * Add deepcopy to allow view.flows.resolve (issue #4916) * try to simplify deepcopy implementation * remove leftover check Co-authored-by: Maximilian Hils --- CHANGELOG.md | 1 + mitmproxy/controller.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cee75fc2df..435dcf9f1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ * Upstream replays don't do CONNECT on plaintext HTTP requests (#4876, @HoffmannP) * Remove workarounds for old pyOpenSSL versions (#4831, @KarlParkinson) * Add fonts to asset filter (~a) (#4928, @elespike) +* Fix bug that crashed when using `view.flows.resolve` (#4916, @rbdixon) ## 28 September 2021: mitmproxy 7.0.4 diff --git a/mitmproxy/controller.py b/mitmproxy/controller.py index 56365362fd..f6b2a22bb0 100644 --- a/mitmproxy/controller.py +++ b/mitmproxy/controller.py @@ -65,6 +65,10 @@ def __del__(self): # This will be ignored by the interpreter, but emit a warning raise exceptions.ControlException(f"Uncommitted reply: {self.obj}") + def __deepcopy__(self, memo): + # some parts of the console ui may use deepcopy, see https://github.com/mitmproxy/mitmproxy/issues/4916 + return memo.setdefault(id(self), DummyReply()) + class DummyReply(Reply): """ From 3a2c87432c927226494064436087ddb779612f82 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 26 Nov 2021 23:14:19 +0100 Subject: [PATCH 032/285] [requires.io] dependency update (#4961) Co-authored-by: requires.io --- tox.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 27ec66667e..c8c3362d9a 100644 --- a/tox.ini +++ b/tox.ini @@ -31,9 +31,9 @@ commands = deps = mypy==0.910 types-certifi==2021.10.8.0 - types-Flask==1.1.5 - types-Werkzeug==1.0.8 - types-requests==2.26.0 + types-Flask==1.1.6 + types-Werkzeug==1.0.9 + types-requests==2.26.1 types-cryptography==3.3.9 types-pyOpenSSL==21.0.1 From 6997129bc07363a69a6414675b195e3e7416827e Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 27 Nov 2021 14:11:23 +0100 Subject: [PATCH 033/285] make sure that `running()` is only invoked once on startup. (#4964) fix #3584 --- CHANGELOG.md | 1 + mitmproxy/addons/script.py | 7 ++++++- test/mitmproxy/addons/test_script.py | 24 +++++++++++++++++++++-- test/mitmproxy/data/addonscripts/error.py | 4 ++-- 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 435dcf9f1c..a07653b406 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ * Remove workarounds for old pyOpenSSL versions (#4831, @KarlParkinson) * Add fonts to asset filter (~a) (#4928, @elespike) * Fix bug that crashed when using `view.flows.resolve` (#4916, @rbdixon) +* Fix a bug where `running()` is invoked twice on startup (#3584, @mhils) ## 28 September 2021: mitmproxy 7.0.4 diff --git a/mitmproxy/addons/script.py b/mitmproxy/addons/script.py index 4e146d67d9..5907476c7f 100644 --- a/mitmproxy/addons/script.py +++ b/mitmproxy/addons/script.py @@ -75,6 +75,7 @@ def __init__(self, path: str, reload: bool) -> None: path.strip("'\" ") ) self.ns = None + self.is_running = False if not os.path.isfile(self.fullpath): raise exceptions.OptionsError('No such script') @@ -85,6 +86,9 @@ def __init__(self, path: str, reload: bool) -> None: else: self.loadscript() + def running(self): + self.is_running = True + def done(self): if self.reloadtask: self.reloadtask.cancel() @@ -105,7 +109,8 @@ def loadscript(self): if self.ns: # We're already running, so we have to explicitly register and # configure the addon - ctx.master.addons.invoke_addon(self.ns, hooks.RunningHook()) + if self.is_running: + ctx.master.addons.invoke_addon(self.ns, hooks.RunningHook()) try: ctx.master.addons.invoke_addon( self.ns, diff --git a/test/mitmproxy/addons/test_script.py b/test/mitmproxy/addons/test_script.py index 12ee070c60..f815f1655a 100644 --- a/test/mitmproxy/addons/test_script.py +++ b/test/mitmproxy/addons/test_script.py @@ -1,5 +1,6 @@ import asyncio import os +import re import sys import traceback @@ -11,6 +12,7 @@ from mitmproxy.proxy.layers.http import HttpRequestHook from mitmproxy.test import taddons from mitmproxy.test import tflow +from mitmproxy.tools import main # We want this to be speedy for testing @@ -90,7 +92,7 @@ async def test_simple(self, tdata): ) with taddons.context(sc) as tctx: tctx.configure(sc) - await tctx.master.await_log("recorder running") + await tctx.master.await_log("recorder configure") rec = tctx.master.addons.get("recorder") assert rec.call_log[0][0:2] == ("recorder", "load") @@ -128,7 +130,7 @@ async def test_exception(self, tdata): True, ) tctx.master.addons.add(sc) - await tctx.master.await_log("error running") + await tctx.master.await_log("error load") tctx.configure(sc) f = tflow.tflow(resp=True) @@ -331,3 +333,21 @@ async def test_order(self, tdata): 'e running', 'e configure', ] + + +def test_order(event_loop, tdata, capsys): + """Integration test: Make sure that the runtime hooks are triggered on startup in the correct order.""" + asyncio.set_event_loop(event_loop) + main.mitmdump([ + "-n", + "-s", tdata.path("mitmproxy/data/addonscripts/recorder/recorder.py"), + "-s", tdata.path("mitmproxy/data/addonscripts/shutdown.py"), + ]) + assert re.match( + r"Loading script.+recorder.py\n" + r"\('recorder', 'load', .+\n" + r"\('recorder', 'configure', .+\n" + r"Loading script.+shutdown.py\n" + r"\('recorder', 'running', .+\n$", + capsys.readouterr().out, + ) diff --git a/test/mitmproxy/data/addonscripts/error.py b/test/mitmproxy/data/addonscripts/error.py index 2f0c1755c2..8dcd404d8b 100644 --- a/test/mitmproxy/data/addonscripts/error.py +++ b/test/mitmproxy/data/addonscripts/error.py @@ -1,8 +1,8 @@ from mitmproxy import ctx -def running(): - ctx.log.info("error running") +def load(loader): + ctx.log.info("error load") def request(flow): From ace07e7e3c70cd1f099705d6d1989bc999ca7ca9 Mon Sep 17 00:00:00 2001 From: James Yale Date: Mon, 20 Dec 2021 19:18:00 +0000 Subject: [PATCH 034/285] Example specified incorrect header (#4997) * Example specified incorrect header * Add CHANGELOG entry reference the documentation update * fixup! Add CHANGELOG entry reference the documentation update --- CHANGELOG.md | 1 + docs/src/content/overview-features.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a07653b406..268574d7b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ * Add fonts to asset filter (~a) (#4928, @elespike) * Fix bug that crashed when using `view.flows.resolve` (#4916, @rbdixon) * Fix a bug where `running()` is invoked twice on startup (#3584, @mhils) +* Correct documentation example for User-Agent header modification (#4997, @jamesyale) ## 28 September 2021: mitmproxy 7.0.4 diff --git a/docs/src/content/overview-features.md b/docs/src/content/overview-features.md index 46ab86507d..ef90aa4305 100644 --- a/docs/src/content/overview-features.md +++ b/docs/src/content/overview-features.md @@ -275,7 +275,7 @@ Set the `User-Agent` header to the data read from `~/useragent.txt` for all requ (existing `User-Agent` headers are replaced): ``` -/~q/Host/@~/useragent.txt +/~q/User-Agent/@~/useragent.txt ``` Remove existing `Host` headers from all requests: From ca646ebd40544d6b73d185218675b95b3ab724ea Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 20 Dec 2021 20:36:40 +0100 Subject: [PATCH 035/285] fix #4957 --- mitmproxy/io/tnetstring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitmproxy/io/tnetstring.py b/mitmproxy/io/tnetstring.py index ac57bf1831..bfe06c0cd0 100644 --- a/mitmproxy/io/tnetstring.py +++ b/mitmproxy/io/tnetstring.py @@ -171,7 +171,7 @@ def load(file_handle: typing.IO[bytes]) -> TSerializable: data_length = b"" while c.isdigit(): data_length += c - if len(data_length) > 9: + if len(data_length) > 12: raise ValueError("not a tnetstring: absurdly large length prefix") c = file_handle.read(1) if c != b":": From c74806feacfe4edcf8ae4a06df0ded9e31704374 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 20 Dec 2021 21:06:14 +0100 Subject: [PATCH 036/285] fix tests --- test/mitmproxy/io/test_tnetstring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/mitmproxy/io/test_tnetstring.py b/test/mitmproxy/io/test_tnetstring.py index 3f5469e5b7..ecca5130bd 100644 --- a/test/mitmproxy/io/test_tnetstring.py +++ b/test/mitmproxy/io/test_tnetstring.py @@ -122,7 +122,7 @@ def test_roundtrip_file_random(self): def test_error_on_absurd_lengths(self): s = io.BytesIO() - s.write(b'1000000000:pwned!,') + s.write(b'1000000000000:pwned!,') s.seek(0) with self.assertRaises(ValueError): tnetstring.load(s) From 96f77453cc5df8c9f239cac66c0ad67009beaf4a Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 20 Dec 2021 16:10:06 -0500 Subject: [PATCH 037/285] fix `change_upstream_proxy.py` example, fix #4981 (#5007) --- examples/contrib/change_upstream_proxy.py | 26 ++++++++++++++++------- mitmproxy/addons/clientplayback.py | 2 +- mitmproxy/proxy/layers/http/__init__.py | 2 +- mitmproxy/proxy/tunnel.py | 7 +++++- 4 files changed, 26 insertions(+), 11 deletions(-) diff --git a/examples/contrib/change_upstream_proxy.py b/examples/contrib/change_upstream_proxy.py index a0e7e57288..cd5841df95 100644 --- a/examples/contrib/change_upstream_proxy.py +++ b/examples/contrib/change_upstream_proxy.py @@ -1,10 +1,18 @@ -from mitmproxy import http import typing +from mitmproxy import http +from mitmproxy.connection import Server +from mitmproxy.net.server_spec import ServerSpec + + # This scripts demonstrates how mitmproxy can switch to a second/different upstream proxy # in upstream proxy mode. # -# Usage: mitmdump -U http://default-upstream-proxy.local:8080/ -s change_upstream_proxy.py +# Usage: mitmdump +# -s change_upstream_proxy.py +# --mode upstream:http://default-upstream-proxy:8080/ +# --set connection_strategy=lazy +# --set upstream_cert=false # # If you want to change the target server, you should modify flow.request.host and flow.request.port @@ -18,10 +26,12 @@ def proxy_address(flow: http.HTTPFlow) -> typing.Tuple[str, int]: def request(flow: http.HTTPFlow) -> None: - if flow.request.method == "CONNECT": - # If the decision is done by domain, one could also modify the server address here. - # We do it after CONNECT here to have the request data available as well. - return address = proxy_address(flow) - if flow.live: - flow.live.change_upstream_proxy_server(address) # type: ignore + + is_proxy_change = address != flow.server_conn.via.address + server_connection_already_open = flow.server_conn.timestamp_start is not None + if is_proxy_change and server_connection_already_open: + # server_conn already refers to an existing connection (which cannot be modified), + # so we need to replace it with a new server connection object. + flow.server_conn = Server(flow.server_conn.address) + flow.server_conn.via = ServerSpec("http", address) diff --git a/mitmproxy/addons/clientplayback.py b/mitmproxy/addons/clientplayback.py index 8dea6f794f..3ae8abad91 100644 --- a/mitmproxy/addons/clientplayback.py +++ b/mitmproxy/addons/clientplayback.py @@ -74,7 +74,7 @@ def __init__(self, flow: http.HTTPFlow, options: Options) -> None: ) context.server.tls = flow.request.scheme == "https" if options.mode.startswith("upstream:"): - context.server.via = server_spec.parse_with_mode(options.mode)[1] + context.server.via = flow.server_conn.via = server_spec.parse_with_mode(options.mode)[1] super().__init__(context) diff --git a/mitmproxy/proxy/layers/http/__init__.py b/mitmproxy/proxy/layers/http/__init__.py index 319b394e86..e73db4b42a 100644 --- a/mitmproxy/proxy/layers/http/__init__.py +++ b/mitmproxy/proxy/layers/http/__init__.py @@ -541,7 +541,7 @@ def make_server_connection(self) -> layer.CommandGenerator[bool]: connection, err = yield GetHttpConnection( (self.flow.request.host, self.flow.request.port), self.flow.request.scheme == "https", - self.context.server.via, + self.flow.server_conn.via, ) if err: yield from self.handle_protocol_error(ResponseProtocolError(self.stream_id, err)) diff --git a/mitmproxy/proxy/tunnel.py b/mitmproxy/proxy/tunnel.py index b033366a0a..ac40062f3f 100644 --- a/mitmproxy/proxy/tunnel.py +++ b/mitmproxy/proxy/tunnel.py @@ -1,3 +1,4 @@ +import time from enum import Enum, auto from typing import List, Optional, Tuple, Union @@ -62,16 +63,20 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: if done: if self.conn != self.tunnel_connection: self.conn.state = connection.ConnectionState.OPEN + self.conn.timestamp_start = time.time() if err: if self.conn != self.tunnel_connection: self.conn.state = connection.ConnectionState.CLOSED + self.conn.timestamp_start = time.time() yield from self.on_handshake_error(err) if done or err: yield from self._handshake_finished(err) else: yield from self.receive_data(event.data) elif isinstance(event, events.ConnectionClosed): - self.conn.state &= ~connection.ConnectionState.CAN_READ + if self.conn != self.tunnel_connection: + self.conn.state &= ~connection.ConnectionState.CAN_READ + self.conn.timestamp_end = time.time() if self.tunnel_state is TunnelState.OPEN: yield from self.receive_close() elif self.tunnel_state is TunnelState.ESTABLISHING: From efd2980b8addf2a524e1d0433eab9cda5ecb2191 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 20 Dec 2021 22:23:23 +0100 Subject: [PATCH 038/285] catch OpenSSL SysCallError, refs #4985 --- mitmproxy/proxy/layers/tls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitmproxy/proxy/layers/tls.py b/mitmproxy/proxy/layers/tls.py index 924a0fff7b..bf9136ccff 100644 --- a/mitmproxy/proxy/layers/tls.py +++ b/mitmproxy/proxy/layers/tls.py @@ -308,7 +308,7 @@ def receive_close(self) -> layer.CommandGenerator[None]: def send_data(self, data: bytes) -> layer.CommandGenerator[None]: try: self.tls.sendall(data) - except SSL.ZeroReturnError: + except (SSL.ZeroReturnError, SSL.SysCallError): # The other peer may still be trying to send data over, which we discard here. pass yield from self.tls_interact() From 9b75f52073aeddcd6fc555b1540574b96317c623 Mon Sep 17 00:00:00 2001 From: Shubhangi Choudhary <78342516+shubhangi013@users.noreply.github.com> Date: Tue, 21 Dec 2021 18:35:55 +0530 Subject: [PATCH 039/285] migrated the files to tsx (#4972) * migrated the files to tsx * reverted unecessary changes * reverted unecessary changes to app.css * fixed the fails * fixed the fails --- web/src/js/components/Modal/Modal.jsx | 13 ------------- web/src/js/components/Modal/Modal.tsx | 13 +++++++++++++ .../Modal/{ModalLayout.jsx => ModalLayout.tsx} | 8 ++++++-- web/src/js/filt/filt.js | 4 +++- 4 files changed, 22 insertions(+), 16 deletions(-) delete mode 100644 web/src/js/components/Modal/Modal.jsx create mode 100644 web/src/js/components/Modal/Modal.tsx rename web/src/js/components/Modal/{ModalLayout.jsx => ModalLayout.tsx} (67%) diff --git a/web/src/js/components/Modal/Modal.jsx b/web/src/js/components/Modal/Modal.jsx deleted file mode 100644 index 64b61fcce8..0000000000 --- a/web/src/js/components/Modal/Modal.jsx +++ /dev/null @@ -1,13 +0,0 @@ -import * as React from "react" -import ModalList from './ModalList' -import { useAppSelector } from "../../ducks"; - - -export default function PureModal() { - const activeModal = useAppSelector(state => state.ui.modal.activeModal) - const ActiveModal = ModalList.find(m => m.name === activeModal ) - - return( - activeModal ? :
- ) -} diff --git a/web/src/js/components/Modal/Modal.tsx b/web/src/js/components/Modal/Modal.tsx new file mode 100644 index 0000000000..d9a475eb91 --- /dev/null +++ b/web/src/js/components/Modal/Modal.tsx @@ -0,0 +1,13 @@ +import * as React from "react" +import ModalList from './ModalList' +import { useAppSelector } from "../../ducks"; + + +export default function PureModal() { + const activeModal : string = useAppSelector(state => state.ui.modal.activeModal) + const ActiveModal:(() => JSX.Element) | undefined= ModalList.find(m => m.name === activeModal ) + + return( + activeModal&&ActiveModal!==undefined ? :
+ ) +} diff --git a/web/src/js/components/Modal/ModalLayout.jsx b/web/src/js/components/Modal/ModalLayout.tsx similarity index 67% rename from web/src/js/components/Modal/ModalLayout.jsx rename to web/src/js/components/Modal/ModalLayout.tsx index 387f7ded97..dd21aed081 100644 --- a/web/src/js/components/Modal/ModalLayout.jsx +++ b/web/src/js/components/Modal/ModalLayout.tsx @@ -1,10 +1,14 @@ import * as React from "react" -export default function ModalLayout ({ children }) { +type ModalLayoutProps = { + children: React.ReactNode, +} + +export default function ModalLayout ({ children}: ModalLayoutProps ) { return (
-