From 7010cf7edb731669e4e492b1a9e791b185c916ee Mon Sep 17 00:00:00 2001
From: kingluo <home_king@163.com>
Date: Tue, 23 Apr 2024 23:32:09 +0800
Subject: [PATCH 1/7] fix(1346): fix control frames flood

part of #1346
---
 fw/http_frame.c | 16 ++++++++++++++++
 fw/http_frame.h | 12 ++++++++++++
 2 files changed, 28 insertions(+)

diff --git a/fw/http_frame.c b/fw/http_frame.c
index 7c2465dba6..5185193c9a 100644
--- a/fw/http_frame.c
+++ b/fw/http_frame.c
@@ -280,6 +280,17 @@ __tfw_h2_send_frame(TfwH2Ctx *ctx, TfwFrameHdr *hdr, TfwStr *data,
 	unsigned char buf[FRAME_HEADER_SIZE];
 	TfwStr *hdr_str = TFW_STR_CHUNK(data, 0);
 	TfwH2Conn *conn = container_of(ctx, TfwH2Conn, h2);
+	bool is_control_frame = !hdr->stream_id || hdr->type == HTTP2_RST_STREAM;
+
+	// If the peer is causing us to generate a lot of control frames,
+	// but not reading them from us, assume they are trying to make us
+	// run out of memory.
+	if (is_control_frame &&
+	    atomic_read(&ctx->queued_control_frames) > MAX_QUEUED_CONTROL_FRAMES) {
+		T_ERR("Too many control frames in send queue, closing connection");
+		r = SS_BLOCK_WITH_RST;
+		goto err;
+	}
 
 	BUG_ON(hdr_str->data);
 	hdr_str->data = buf;
@@ -324,6 +335,11 @@ __tfw_h2_send_frame(TfwH2Ctx *ctx, TfwFrameHdr *hdr, TfwStr *data,
 			tfw_h2_on_tcp_entail_ack;
 	}
 
+	if (is_control_frame) {
+		skb_set_tfw_flags(it.skb, SS_F_HTTT2_FRAME_CONTROL);
+		atomic_inc(&ctx->queued_control_frames);
+	}
+
 	if ((r = tfw_connection_send((TfwConn *)conn, &msg)))
 		goto err;
 	/*
diff --git a/fw/http_frame.h b/fw/http_frame.h
index bca9e1f1ba..ed185e2797 100644
--- a/fw/http_frame.h
+++ b/fw/http_frame.h
@@ -31,6 +31,18 @@
 #define FRAME_MAX_LENGTH		((1U << 24) - 1)
 #define FRAME_DEF_LENGTH		(16384)
 
+/**
+ * MAX_QUEUED_CONTROL_FRAMES is the maximum number of control frames like
+ * SETTINGS, PING and RST_STREAM that will be queued for writing before
+ * the connection is closed to prevent memory exhaustion attacks.
+ */
+#define MAX_QUEUED_CONTROL_FRAMES	10000
+
+enum {
+	/* This skb contains control frame. */
+	SS_F_HTTT2_FRAME_CONTROL	= 0x80,
+};
+
 /**
  * HTTP/2 frame types (RFC 7540 section 6).
  */

From a106b7ed6f7936c2c029c1b0225895db1429a5cc Mon Sep 17 00:00:00 2001
From: kingluo <home_king@163.com>
Date: Mon, 6 May 2024 17:44:30 +0800
Subject: [PATCH 2/7] move MAX_QUEUED_CONTROL_FRAMES to configuration file

---
 etc/tempesta_fw.conf | 13 +++++++++++++
 fw/http.c            | 35 +++++++++++++++++++++++++++++++++++
 fw/http_frame.c      | 13 +++++++++----
 fw/http_frame.h      |  2 ++
 4 files changed, 59 insertions(+), 4 deletions(-)

diff --git a/etc/tempesta_fw.conf b/etc/tempesta_fw.conf
index 9e074d4fb9..1778f2b2a2 100644
--- a/etc/tempesta_fw.conf
+++ b/etc/tempesta_fw.conf
@@ -1149,6 +1149,19 @@
 #   http_max_header_list_size 4096;
 #
 
+# TAG: max_queued_control_frames
+#
+# Maximum number of control frames allowed to wait in
+# the client connection's write queue, to mitigate control frames flooding
+# (PING/SETTINGS/RESET_STREAM).
+#
+# Syntax:
+#   max_queued_control_frames SIZE
+#
+# Example:
+#   max_queued_control_frames 10000;
+#
+
 #
 # Health monitoring configuration.
 #
diff --git a/fw/http.c b/fw/http.c
index 7eabcde5fd..c5157dd73f 100644
--- a/fw/http.c
+++ b/fw/http.c
@@ -7574,6 +7574,34 @@ tfw_cfgop_cleanup_max_header_list_size(TfwCfgSpec *cs)
 	max_header_list_size = 0;
 }
 
+static int
+tfw_cfgop_max_queued_control_frames(TfwCfgSpec *cs, TfwCfgEntry *ce)
+{
+	int r;
+
+	if (tfw_cfg_check_val_n(ce, 1))
+		return -EINVAL;
+	if (ce->attr_n) {
+		T_ERR_NL("Unexpected attributes\n");
+		return -EINVAL;
+	}
+
+	r = tfw_cfg_parse_uint(ce->vals[0], &max_queued_control_frames);
+	if (unlikely(r)) {
+		T_ERR_NL("Unable to parse 'max_queued_control_frames' value: '%s'\n",
+			 ce->vals[0]);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void
+tfw_cfgop_cleanup_max_queued_control_frames(TfwCfgSpec *cs)
+{
+	max_queued_control_frames = 0;
+}
+
 static TfwCfgSpec tfw_http_specs[] = {
 	{
 		.name = "block_action",
@@ -7661,6 +7689,13 @@ static TfwCfgSpec tfw_http_specs[] = {
 		.allow_none = true,
 		.cleanup = tfw_cfgop_cleanup_max_header_list_size,
 	},
+	{
+		.name = "max_queued_control_frames",
+		.deflt = "10000",
+		.handler = tfw_cfgop_max_queued_control_frames,
+		.allow_none = true,
+		.cleanup = tfw_cfgop_cleanup_max_queued_control_frames,
+	},
 	{ 0 }
 };
 
diff --git a/fw/http_frame.c b/fw/http_frame.c
index 5185193c9a..98260d0712 100644
--- a/fw/http_frame.c
+++ b/fw/http_frame.c
@@ -31,6 +31,8 @@
 #include "http_msg.h"
 #include "tcp.h"
 
+unsigned int max_queued_control_frames = 0;
+
 #define FRAME_PREFACE_CLI_MAGIC		"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"
 #define FRAME_PREFACE_CLI_MAGIC_LEN	24
 #define FRAME_WND_UPDATE_SIZE		4
@@ -282,11 +284,14 @@ __tfw_h2_send_frame(TfwH2Ctx *ctx, TfwFrameHdr *hdr, TfwStr *data,
 	TfwH2Conn *conn = container_of(ctx, TfwH2Conn, h2);
 	bool is_control_frame = !hdr->stream_id || hdr->type == HTTP2_RST_STREAM;
 
-	// If the peer is causing us to generate a lot of control frames,
-	// but not reading them from us, assume they are trying to make us
-	// run out of memory.
+	/*
+	 * If the peer is causing us to generate a lot of control frames,
+	 * but not reading them from us, assume they are trying to make us
+	 * run out of memory.
+	 */
 	if (is_control_frame &&
-	    atomic_read(&ctx->queued_control_frames) > MAX_QUEUED_CONTROL_FRAMES) {
+	    atomic_read(&ctx->queued_control_frames)
+	                > max_queued_control_frames) {
 		T_ERR("Too many control frames in send queue, closing connection");
 		r = SS_BLOCK_WITH_RST;
 		goto err;
diff --git a/fw/http_frame.h b/fw/http_frame.h
index ed185e2797..fa5cfbb783 100644
--- a/fw/http_frame.h
+++ b/fw/http_frame.h
@@ -24,6 +24,8 @@
 #include "http_stream.h"
 #include "hpack.h"
 
+extern unsigned int max_queued_control_frames;
+
 /* RFC 7540 Section 4.1 frame header constants. */
 #define FRAME_HEADER_SIZE		9
 #define FRAME_STREAM_ID_MASK		((1U << 31) - 1)

From 685b84baa4091e4bf157b7815e1fd5d1cae68ddc Mon Sep 17 00:00:00 2001
From: kingluo <home_king@163.com>
Date: Wed, 15 May 2024 14:38:03 +0800
Subject: [PATCH 3/7] fix PR

---
 fw/http_frame.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/fw/http_frame.c b/fw/http_frame.c
index 98260d0712..adb62e773b 100644
--- a/fw/http_frame.c
+++ b/fw/http_frame.c
@@ -290,9 +290,9 @@ __tfw_h2_send_frame(TfwH2Ctx *ctx, TfwFrameHdr *hdr, TfwStr *data,
 	 * run out of memory.
 	 */
 	if (is_control_frame &&
-	    atomic_read(&ctx->queued_control_frames)
-	                > max_queued_control_frames) {
-		T_ERR("Too many control frames in send queue, closing connection");
+	    ctx->queued_control_frames > max_queued_control_frames)
+	{
+		T_WARN("Too many control frames in send queue, closing connection");
 		r = SS_BLOCK_WITH_RST;
 		goto err;
 	}
@@ -342,7 +342,7 @@ __tfw_h2_send_frame(TfwH2Ctx *ctx, TfwFrameHdr *hdr, TfwStr *data,
 
 	if (is_control_frame) {
 		skb_set_tfw_flags(it.skb, SS_F_HTTT2_FRAME_CONTROL);
-		atomic_inc(&ctx->queued_control_frames);
+		++ctx->queued_control_frames;
 	}
 
 	if ((r = tfw_connection_send((TfwConn *)conn, &msg)))

From c3f311cf257e4026df00d33c6085dd2c04e6b764 Mon Sep 17 00:00:00 2001
From: kingluo <home_king@163.com>
Date: Mon, 3 Jun 2024 21:31:37 +0800
Subject: [PATCH 4/7] fix T_WARN

---
 fw/http_frame.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fw/http_frame.c b/fw/http_frame.c
index adb62e773b..0354728fdb 100644
--- a/fw/http_frame.c
+++ b/fw/http_frame.c
@@ -292,7 +292,7 @@ __tfw_h2_send_frame(TfwH2Ctx *ctx, TfwFrameHdr *hdr, TfwStr *data,
 	if (is_control_frame &&
 	    ctx->queued_control_frames > max_queued_control_frames)
 	{
-		T_WARN("Too many control frames in send queue, closing connection");
+		T_WARN("Too many control frames in send queue, closing connection\n");
 		r = SS_BLOCK_WITH_RST;
 		goto err;
 	}

From 1195e1eb4c8e009e03846ec8f04ee794d654104a Mon Sep 17 00:00:00 2001
From: kingluo <home_king@163.com>
Date: Wed, 10 Jul 2024 15:37:13 +0800
Subject: [PATCH 5/7] rework based on tfw_tls_encrypt and skb->cb

---
 fw/http_frame.c |  2 +-
 fw/http_frame.h |  7 +------
 fw/tls.c        | 11 ++++++++++-
 3 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/fw/http_frame.c b/fw/http_frame.c
index 0354728fdb..9211401af0 100644
--- a/fw/http_frame.c
+++ b/fw/http_frame.c
@@ -341,7 +341,7 @@ __tfw_h2_send_frame(TfwH2Ctx *ctx, TfwFrameHdr *hdr, TfwStr *data,
 	}
 
 	if (is_control_frame) {
-		skb_set_tfw_flags(it.skb, SS_F_HTTT2_FRAME_CONTROL);
+		it.skb->cb[SKB_CB_FLAGS_IDX] |= SS_F_HTTT2_FRAME_CONTROL;
 		++ctx->queued_control_frames;
 	}
 
diff --git a/fw/http_frame.h b/fw/http_frame.h
index fa5cfbb783..d0db61baca 100644
--- a/fw/http_frame.h
+++ b/fw/http_frame.h
@@ -33,12 +33,7 @@ extern unsigned int max_queued_control_frames;
 #define FRAME_MAX_LENGTH		((1U << 24) - 1)
 #define FRAME_DEF_LENGTH		(16384)
 
-/**
- * MAX_QUEUED_CONTROL_FRAMES is the maximum number of control frames like
- * SETTINGS, PING and RST_STREAM that will be queued for writing before
- * the connection is closed to prevent memory exhaustion attacks.
- */
-#define MAX_QUEUED_CONTROL_FRAMES	10000
+#define SKB_CB_FLAGS_IDX		47
 
 enum {
 	/* This skb contains control frame. */
diff --git a/fw/tls.c b/fw/tls.c
index 51d9ebcb23..4fd0167399 100644
--- a/fw/tls.c
+++ b/fw/tls.c
@@ -3,7 +3,7 @@
  *
  * Transport Layer Security (TLS) interfaces to Tempesta TLS.
  *
- * Copyright (C) 2015-2023 Tempesta Technologies, Inc.
+ * Copyright (C) 2015-2024 Tempesta Technologies, Inc.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by
@@ -262,6 +262,8 @@ tfw_tls_encrypt(struct sock *sk, struct sk_buff *skb, unsigned int mss_now,
 	struct scatterlist sg[AUTO_SEGS_N], out_sg[AUTO_SEGS_N];
 	struct page **pages = NULL, **pages_end, **p;
 	struct page *auto_pages[AUTO_SEGS_N];
+	TfwConn *conn = sk->sk_user_data;
+	TfwH2Ctx *h2;
 
 	tls = tfw_tls_context(sk->sk_user_data);
 	io = &tls->io_out;
@@ -320,6 +322,13 @@ tfw_tls_encrypt(struct sock *sk, struct sk_buff *skb, unsigned int mss_now,
 		sgt.nents += next_nents;
 		out_sgt.nents += next_nents;
 		skb_tail = next;
+
+		if (TFW_CONN_PROTO(conn) == TFW_FSM_H2) {
+			h2 = tfw_h2_context(conn);
+			if (unlikely(skb->cb[SKB_CB_FLAGS_IDX] & SS_F_HTTT2_FRAME_CONTROL)) {
+				--h2->queued_control_frames;
+			}
+		}
 	}
 
 	len += head_sz + TTLS_TAG_LEN;

From 902025aece5c030becedeca85a5535a52b7555eb Mon Sep 17 00:00:00 2001
From: kingluo <home_king@163.com>
Date: Wed, 10 Jul 2024 19:43:10 +0800
Subject: [PATCH 6/7] rebase fix

---
 fw/http2.h | 1 +
 fw/tls.c   | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/fw/http2.h b/fw/http2.h
index 69e5a55dcc..e6239ed174 100644
--- a/fw/http2.h
+++ b/fw/http2.h
@@ -109,6 +109,7 @@ typedef struct {
  */
 typedef struct tfw_h2_ctx_t {
         spinlock_t      lock;
+        unsigned int    queued_control_frames;
         TfwSettings     lsettings;
         TfwSettings     rsettings;
         unsigned int    lstream_id;
diff --git a/fw/tls.c b/fw/tls.c
index 4fd0167399..39000e819c 100644
--- a/fw/tls.c
+++ b/fw/tls.c
@@ -324,7 +324,7 @@ tfw_tls_encrypt(struct sock *sk, struct sk_buff *skb, unsigned int mss_now,
 		skb_tail = next;
 
 		if (TFW_CONN_PROTO(conn) == TFW_FSM_H2) {
-			h2 = tfw_h2_context(conn);
+			h2 = tfw_h2_context_safe(conn);
 			if (unlikely(skb->cb[SKB_CB_FLAGS_IDX] & SS_F_HTTT2_FRAME_CONTROL)) {
 				--h2->queued_control_frames;
 			}

From 6f5b30d30f0000037f289cf9bb126db7df8699d4 Mon Sep 17 00:00:00 2001
From: kingluo <home_king@163.com>
Date: Thu, 11 Jul 2024 00:37:14 +0800
Subject: [PATCH 7/7] store flags in tcp_skb_cb->unused

---
 fw/http_frame.c |  2 +-
 fw/http_frame.h |  5 ++---
 fw/ss_skb.c     |  7 ++++++-
 fw/ss_skb.h     |  4 +++-
 fw/tls.c        | 25 +++++++++++++++++--------
 5 files changed, 29 insertions(+), 14 deletions(-)

diff --git a/fw/http_frame.c b/fw/http_frame.c
index 9211401af0..c43b56197a 100644
--- a/fw/http_frame.c
+++ b/fw/http_frame.c
@@ -341,7 +341,7 @@ __tfw_h2_send_frame(TfwH2Ctx *ctx, TfwFrameHdr *hdr, TfwStr *data,
 	}
 
 	if (is_control_frame) {
-		it.skb->cb[SKB_CB_FLAGS_IDX] |= SS_F_HTTT2_FRAME_CONTROL;
+		TFW_SKB_CB(msg.skb_head)->is_control_frame = true;
 		++ctx->queued_control_frames;
 	}
 
diff --git a/fw/http_frame.h b/fw/http_frame.h
index d0db61baca..cfc8fbf853 100644
--- a/fw/http_frame.h
+++ b/fw/http_frame.h
@@ -33,11 +33,10 @@ extern unsigned int max_queued_control_frames;
 #define FRAME_MAX_LENGTH		((1U << 24) - 1)
 #define FRAME_DEF_LENGTH		(16384)
 
-#define SKB_CB_FLAGS_IDX		47
-
+/* flags in TCP_SKB_CB(skb)->unused, only 5 bits available */
 enum {
 	/* This skb contains control frame. */
-	SS_F_HTTT2_FRAME_CONTROL	= 0x80,
+	SS_F_HTTT2_FRAME_CONTROL	= 0x01,
 };
 
 /**
diff --git a/fw/ss_skb.c b/fw/ss_skb.c
index 3bf00b49a5..d918c457c6 100644
--- a/fw/ss_skb.c
+++ b/fw/ss_skb.c
@@ -7,7 +7,7 @@
  * on top on native Linux socket buffers. The helpers provide common and
  * convenient wrappers for skb processing.
  *
- * Copyright (C) 2015-2023 Tempesta Technologies, Inc.
+ * Copyright (C) 2015-2024 Tempesta Technologies, Inc.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by
@@ -1309,6 +1309,7 @@ ss_skb_init_for_xmit(struct sk_buff *skb)
 {
 	struct skb_shared_info *shinfo = skb_shinfo(skb);
 	__u8 pfmemalloc = skb->pfmemalloc;
+	bool is_control_frame = TFW_SKB_CB(skb)->is_control_frame;
 
 	WARN_ON_ONCE(skb->sk);
 
@@ -1320,6 +1321,10 @@ ss_skb_init_for_xmit(struct sk_buff *skb)
 	 */
 	memset(skb->cb, 0, sizeof(skb->cb));
 
+	/* Reserve unused bits as http2 flags */
+	if (is_control_frame)
+		TCP_SKB_CB(skb)->unused |= SS_F_HTTT2_FRAME_CONTROL;
+
 	if (!skb_transport_header_was_set(skb)) {
 		/* Quick path for new skbs. */
 		skb->ip_summed = CHECKSUM_PARTIAL;
diff --git a/fw/ss_skb.h b/fw/ss_skb.h
index d815df0f22..8b9d861e08 100644
--- a/fw/ss_skb.h
+++ b/fw/ss_skb.h
@@ -3,7 +3,7 @@
  *
  * Synchronous Sockets API for Linux socket buffers manipulation.
  *
- * Copyright (C) 2015-2023 Tempesta Technologies, Inc.
+ * Copyright (C) 2015-2024 Tempesta Technologies, Inc.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by
@@ -65,6 +65,7 @@ typedef void (*on_tcp_entail_t)(void *conn, struct sk_buff *skb_head);
  *                        to socket write queue;
  * @stream_id		- id of sender stream;
  * @is_head		- flag indicates that this is a head of skb list;
+ * @is_control_frame	- flag indicates that this is a h2 control frame;
  */
 struct tfw_skb_cb {
 	void 		*opaque_data;
@@ -73,6 +74,7 @@ struct tfw_skb_cb {
 	on_tcp_entail_t on_tcp_entail;
 	unsigned int 	stream_id;
 	bool		is_head;
+	bool		is_control_frame;
 };
 
 #define TFW_SKB_CB(skb) ((struct tfw_skb_cb *)&((skb)->cb[0]))
diff --git a/fw/tls.c b/fw/tls.c
index 39000e819c..4e719c8c9f 100644
--- a/fw/tls.c
+++ b/fw/tls.c
@@ -246,6 +246,19 @@ tfw_tls_encrypt(struct sock *sk, struct sk_buff *skb, unsigned int mss_now,
 #define AUTO_SEGS_N	8
 #define MAX_SEG_N	64
 
+#define DEC_CONTROL_FRAME_COUNTER(skb) \
+do { \
+	TfwConn *conn = sk->sk_user_data; \
+	if (TFW_CONN_PROTO(conn) == TFW_FSM_H2) { \
+		TfwH2Ctx *h2 = tfw_h2_context_safe(conn); \
+		if (unlikely(TCP_SKB_CB(skb)->unused & SS_F_HTTT2_FRAME_CONTROL)) { \
+			TCP_SKB_CB(skb)->unused &= ~SS_F_HTTT2_FRAME_CONTROL; \
+			BUG_ON(h2->queued_control_frames == 0); \
+			--h2->queued_control_frames; \
+		} \
+	} \
+} while(0);
+
 	int r = -ENOMEM;
 	unsigned int head_sz, len, frags, t_sz, out_frags, next_nents;
 	unsigned char type;
@@ -262,8 +275,6 @@ tfw_tls_encrypt(struct sock *sk, struct sk_buff *skb, unsigned int mss_now,
 	struct scatterlist sg[AUTO_SEGS_N], out_sg[AUTO_SEGS_N];
 	struct page **pages = NULL, **pages_end, **p;
 	struct page *auto_pages[AUTO_SEGS_N];
-	TfwConn *conn = sk->sk_user_data;
-	TfwH2Ctx *h2;
 
 	tls = tfw_tls_context(sk->sk_user_data);
 	io = &tls->io_out;
@@ -323,16 +334,13 @@ tfw_tls_encrypt(struct sock *sk, struct sk_buff *skb, unsigned int mss_now,
 		out_sgt.nents += next_nents;
 		skb_tail = next;
 
-		if (TFW_CONN_PROTO(conn) == TFW_FSM_H2) {
-			h2 = tfw_h2_context_safe(conn);
-			if (unlikely(skb->cb[SKB_CB_FLAGS_IDX] & SS_F_HTTT2_FRAME_CONTROL)) {
-				--h2->queued_control_frames;
-			}
-		}
+		DEC_CONTROL_FRAME_COUNTER(next);
 	}
 
 	len += head_sz + TTLS_TAG_LEN;
 
+	DEC_CONTROL_FRAME_COUNTER(skb);
+
 	/*
 	 * Use skb_tail->next as skb_head in __extend_pgfrags() to not try to
 	 * put TAG to the next skb, which is out of our limit. In worst case,
@@ -501,6 +509,7 @@ tfw_tls_encrypt(struct sock *sk, struct sk_buff *skb, unsigned int mss_now,
 	return r;
 #undef AUTO_SEGS_N
 #undef MAX_SEG_N
+#undef DEC_CONTROL_FRAME_COUNTER
 }
 
 static inline int