From b828f4993ad7b367a79915ea171e379ecfd2635a Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sun, 22 Oct 2023 13:58:03 +1030 Subject: [PATCH 01/15] Makefile: fix update-mocks dependencies. It definitely needs generated source files. Signed-off-by: Rusty Russell --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d11c127292ef..7e3f4fb68a99 100644 --- a/Makefile +++ b/Makefile @@ -734,7 +734,7 @@ update-mocks: $(ALL_TEST_PROGRAMS:%=update-mocks/%.c) $(ALL_TEST_PROGRAMS:%=update-mocks/%.c): $(ALL_GEN_HEADERS) $(EXTERNAL_LIBS) libccan.a ccan/ccan/cdump/tools/cdump-enumstr config.vars -update-mocks/%: % +update-mocks/%: % $(ALL_GEN_HEADERS) $(ALL_GEN_SOURCES) @MAKE=$(MAKE) tools/update-mocks.sh "$*" $(SUPPRESS_OUTPUT) unittest/%: % From 0dd17fea3c434d8aabd4a9d390d9f1938c39f44c Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sun, 22 Oct 2023 14:37:21 +1030 Subject: [PATCH 02/15] tools: make sure we don't use optimization when building mocks. Don't want it to optimize something out! Signed-off-by: Rusty Russell --- tools/update-mocks.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/update-mocks.sh b/tools/update-mocks.sh index 140ca7717d26..1239c60440d7 100755 --- a/tools/update-mocks.sh +++ b/tools/update-mocks.sh @@ -13,7 +13,8 @@ START=$(grep -F -n '/* AUTOGENERATED MOCKS START */' "$FILE" | cut -d: -f1) END=$(grep -F -n '/* AUTOGENERATED MOCKS END */' "$FILE" | cut -d: -f1) function make_binary() { - $MAKE SUPPRESS_GENERATION=1 "${FILE/%.c/}" 2> "${BASE}.err" >/dev/null + # Make sure we don't optimize out options, and we use debug build + $MAKE SUPPRESS_GENERATION=1 "${FILE/%.c/}" DEBUGBUILD=1 COPTFLAGS="" 2> "${BASE}.err" >/dev/null } if [ -n "$START" ]; then From f6fe3c582935b74d26fd0a571b5d87c4deaeb5ee Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sun, 22 Oct 2023 14:37:31 +1030 Subject: [PATCH 03/15] lightningd: stop all subds when we want to disconnect. This prepares us for connectd disconnecting more gently: it can wait for these to all disconnect. Signed-off-by: Rusty Russell --- lightningd/channel.c | 8 ++---- lightningd/channel_control.c | 7 ++--- lightningd/closing_control.c | 6 ++-- lightningd/connect_control.c | 31 +++++++++++++++++++++ lightningd/connect_control.h | 5 ++++ lightningd/dual_open_control.c | 5 ++-- lightningd/peer_control.c | 4 +-- lightningd/test/run-invoice-select-inchan.c | 8 ++++-- wallet/test/run-wallet.c | 8 ++++-- 9 files changed, 58 insertions(+), 24 deletions(-) diff --git a/lightningd/channel.c b/lightningd/channel.c index 976d690a2b0e..d595d03dc08b 100644 --- a/lightningd/channel.c +++ b/lightningd/channel.c @@ -1023,11 +1023,9 @@ static void channel_err(struct channel *channel, bool disconnect, const char *wh channel_set_owner(channel, NULL); /* Force a disconnect in case the issue is with TCP */ - if (disconnect && channel->peer->ld->connectd) { - const struct peer *peer = channel->peer; - subd_send_msg(peer->ld->connectd, - take(towire_connectd_discard_peer(NULL, &peer->id, - peer->connectd_counter))); + if (disconnect) { + force_peer_disconnect(channel->peer->ld, channel->peer, + "One channel had an error"); } } diff --git a/lightningd/channel_control.c b/lightningd/channel_control.c index 0682f9f7a8c4..6725e43048f7 100644 --- a/lightningd/channel_control.c +++ b/lightningd/channel_control.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -1371,10 +1372,8 @@ bool peer_start_channeld(struct channel *channel, if (!channel->owner) { log_broken(channel->log, "Could not subdaemon channel: %s", strerror(errno)); - /* Disconnect it. */ - subd_send_msg(ld->connectd, - take(towire_connectd_discard_peer(NULL, &channel->peer->id, - channel->peer->connectd_counter))); + force_peer_disconnect(ld, channel->peer, + "Failed to create channeld"); return false; } diff --git a/lightningd/closing_control.c b/lightningd/closing_control.c index c0b8997b0e6b..0547f06d42a7 100644 --- a/lightningd/closing_control.c +++ b/lightningd/closing_control.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -397,9 +398,8 @@ void peer_start_closingd(struct channel *channel, struct peer_fd *peer_fd) log_broken(channel->log, "Could not subdaemon closing: %s", strerror(errno)); /* Disconnect it. */ - subd_send_msg(ld->connectd, - take(towire_connectd_discard_peer(NULL, &channel->peer->id, - channel->peer->connectd_counter))); + force_peer_disconnect(ld, channel->peer, + "Failed to create closingd"); return; } diff --git a/lightningd/connect_control.c b/lightningd/connect_control.c index de7f2b4d59d9..3633d33ab8ad 100644 --- a/lightningd/connect_control.c +++ b/lightningd/connect_control.c @@ -614,6 +614,37 @@ static unsigned connectd_msg(struct subd *connectd, const u8 *msg, const int *fd return 0; } +void force_peer_disconnect(struct lightningd *ld, + const struct peer *peer, + const char *why) +{ + struct channel *c, *next; + + /* Don't bother on shutting down */ + if (!ld->connectd) + return; + + /* Disconnect subds */ + if (peer->uncommitted_channel) + kill_uncommitted_channel(peer->uncommitted_channel, why); + + list_for_each_safe(&peer->channels, c, next, list) { + if (!c->owner) + continue; + + log_debug(c->log, "Forcing disconnect due to %s", why); + /* This frees c! */ + if (channel_state_uncommitted(c->state)) + channel_unsaved_close_conn(c, why); + else + channel_set_owner(c, NULL); + } + + subd_send_msg(peer->ld->connectd, + take(towire_connectd_discard_peer(NULL, &peer->id, + peer->connectd_counter))); +} + static void connect_init_done(struct subd *connectd, const u8 *reply, const int *fds UNUSED, diff --git a/lightningd/connect_control.h b/lightningd/connect_control.h index 781a69f97582..a6819f1fbb29 100644 --- a/lightningd/connect_control.h +++ b/lightningd/connect_control.h @@ -15,6 +15,11 @@ int connectd_init(struct lightningd *ld); void connectd_activate(struct lightningd *ld); void connectd_start_shutdown(struct subd *connectd); +/* Kill subds, tell connectd to disconnect once they're drained. */ +void force_peer_disconnect(struct lightningd *ld, + const struct peer *peer, + const char *why); + void try_reconnect(const tal_t *ctx, struct peer *peer, const struct wireaddr_internal *addrhint); diff --git a/lightningd/dual_open_control.c b/lightningd/dual_open_control.c index 85d560c1229b..83e2abf1a14b 100644 --- a/lightningd/dual_open_control.c +++ b/lightningd/dual_open_control.c @@ -3873,9 +3873,8 @@ bool peer_restart_dualopend(struct peer *peer, log_broken(channel->log, "Could not subdaemon channel: %s", strerror(errno)); /* Disconnect it. */ - subd_send_msg(peer->ld->connectd, - take(towire_connectd_discard_peer(NULL, &channel->peer->id, - channel->peer->connectd_counter))); + force_peer_disconnect(peer->ld, peer, + "Failed to create dualopend"); return false; } diff --git a/lightningd/peer_control.c b/lightningd/peer_control.c index c7417e449e14..2774a570aab6 100644 --- a/lightningd/peer_control.c +++ b/lightningd/peer_control.c @@ -2431,9 +2431,7 @@ static struct command_result *json_disconnect(struct command *cmd, channel_state_name(channel)); } - subd_send_msg(peer->ld->connectd, - take(towire_connectd_discard_peer(NULL, &peer->id, - peer->connectd_counter))); + force_peer_disconnect(cmd->ld, peer, "disconnect command"); /* Connectd tells us when it's finally disconnected */ dc = tal(cmd, struct disconnect_command); diff --git a/lightningd/test/run-invoice-select-inchan.c b/lightningd/test/run-invoice-select-inchan.c index bd6080970094..6c13a9829c2a 100644 --- a/lightningd/test/run-invoice-select-inchan.c +++ b/lightningd/test/run-invoice-select-inchan.c @@ -255,6 +255,11 @@ struct plugin *find_plugin_for_command(struct lightningd *ld UNNEEDED, /* Generated stub for fixup_htlcs_out */ void fixup_htlcs_out(struct lightningd *ld UNNEEDED) { fprintf(stderr, "fixup_htlcs_out called!\n"); abort(); } +/* Generated stub for force_peer_disconnect */ +void force_peer_disconnect(struct lightningd *ld UNNEEDED, + const struct peer *peer UNNEEDED, + const char *why UNNEEDED) +{ fprintf(stderr, "force_peer_disconnect called!\n"); abort(); } /* Generated stub for fromwire_bigsize */ bigsize_t fromwire_bigsize(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) { fprintf(stderr, "fromwire_bigsize called!\n"); abort(); } @@ -885,9 +890,6 @@ u8 *towire_channeld_dev_memleak(const tal_t *ctx UNNEEDED) /* Generated stub for towire_channeld_dev_reenable_commit */ u8 *towire_channeld_dev_reenable_commit(const tal_t *ctx UNNEEDED) { fprintf(stderr, "towire_channeld_dev_reenable_commit called!\n"); abort(); } -/* Generated stub for towire_connectd_discard_peer */ -u8 *towire_connectd_discard_peer(const tal_t *ctx UNNEEDED, const struct node_id *id UNNEEDED, u64 counter UNNEEDED) -{ fprintf(stderr, "towire_connectd_discard_peer called!\n"); abort(); } /* Generated stub for towire_connectd_peer_connect_subd */ u8 *towire_connectd_peer_connect_subd(const tal_t *ctx UNNEEDED, const struct node_id *id UNNEEDED, u64 counter UNNEEDED, const struct channel_id *channel_id UNNEEDED) { fprintf(stderr, "towire_connectd_peer_connect_subd called!\n"); abort(); } diff --git a/wallet/test/run-wallet.c b/wallet/test/run-wallet.c index c76624d8b90c..54190c33dd38 100644 --- a/wallet/test/run-wallet.c +++ b/wallet/test/run-wallet.c @@ -165,6 +165,11 @@ void fatal(const char *fmt UNNEEDED, ...) /* Generated stub for fatal_vfmt */ void fatal_vfmt(const char *fmt UNNEEDED, va_list ap UNNEEDED) { fprintf(stderr, "fatal_vfmt called!\n"); abort(); } +/* Generated stub for force_peer_disconnect */ +void force_peer_disconnect(struct lightningd *ld UNNEEDED, + const struct peer *peer UNNEEDED, + const char *why UNNEEDED) +{ fprintf(stderr, "force_peer_disconnect called!\n"); abort(); } /* Generated stub for fromwire_channeld_dev_memleak_reply */ bool fromwire_channeld_dev_memleak_reply(const void *p UNNEEDED, bool *leak UNNEEDED) { fprintf(stderr, "fromwire_channeld_dev_memleak_reply called!\n"); abort(); } @@ -769,9 +774,6 @@ u8 *towire_channeld_offer_htlc(const tal_t *ctx UNNEEDED, struct amount_msat amo /* Generated stub for towire_channeld_sending_commitsig_reply */ u8 *towire_channeld_sending_commitsig_reply(const tal_t *ctx UNNEEDED) { fprintf(stderr, "towire_channeld_sending_commitsig_reply called!\n"); abort(); } -/* Generated stub for towire_connectd_discard_peer */ -u8 *towire_connectd_discard_peer(const tal_t *ctx UNNEEDED, const struct node_id *id UNNEEDED, u64 counter UNNEEDED) -{ fprintf(stderr, "towire_connectd_discard_peer called!\n"); abort(); } /* Generated stub for towire_connectd_peer_connect_subd */ u8 *towire_connectd_peer_connect_subd(const tal_t *ctx UNNEEDED, const struct node_id *id UNNEEDED, u64 counter UNNEEDED, const struct channel_id *channel_id UNNEEDED) { fprintf(stderr, "towire_connectd_peer_connect_subd called!\n"); abort(); } From b374f3ce0c82fdddd69cad3e157b0280b3ef93ad Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sun, 22 Oct 2023 14:37:31 +1030 Subject: [PATCH 04/15] connectd: give subds a chance to drain when lightningd says to disconnect. Signed-off-by: Rusty Russell --- connectd/connectd.c | 4 +++- connectd/multiplex.c | 2 +- connectd/multiplex.h | 4 ++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/connectd/connectd.c b/connectd/connectd.c index 587faf87c3aa..54442361a1b9 100644 --- a/connectd/connectd.c +++ b/connectd/connectd.c @@ -1812,8 +1812,10 @@ static void peer_discard(struct daemon *daemon, const u8 *msg) /* If it's reconnected already, it will learn soon. */ if (peer->counter != counter) return; + + /* We make sure any final messages from the subds are sent! */ status_peer_debug(&id, "discard_peer"); - tal_free(peer); + drain_peer(peer); } static void start_shutdown(struct daemon *daemon, const u8 *msg) diff --git a/connectd/multiplex.c b/connectd/multiplex.c index 2a7a29529676..d24a23f5fb63 100644 --- a/connectd/multiplex.c +++ b/connectd/multiplex.c @@ -108,7 +108,7 @@ static void close_subd_timeout(struct subd *subd) io_close(subd->conn); } -static void drain_peer(struct peer *peer) +void drain_peer(struct peer *peer) { status_debug("drain_peer"); assert(!peer->draining); diff --git a/connectd/multiplex.h b/connectd/multiplex.h index 77a41fa2aae4..cc4beebadb8a 100644 --- a/connectd/multiplex.h +++ b/connectd/multiplex.h @@ -34,4 +34,8 @@ void send_custommsg(struct daemon *daemon, const u8 *msg); /* Lightningd wants to talk to you. */ void peer_connect_subd(struct daemon *daemon, const u8 *msg, int fd); + +/* Start shutting down peer. */ +void drain_peer(struct peer *peer); + #endif /* LIGHTNING_CONNECTD_MULTIPLEX_H */ From d7967112589d6068fd677fffcecddbcddf4d5897 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sun, 22 Oct 2023 14:37:31 +1030 Subject: [PATCH 05/15] connectd: don't disconnect automatically on sending warning or error. This changes some tests which expected us to disconnect. Signed-off-by: Rusty Russell --- connectd/multiplex.c | 17 -------------- tests/test_connection.py | 48 ++++++++++++---------------------------- tests/test_plugin.py | 6 +---- 3 files changed, 15 insertions(+), 56 deletions(-) diff --git a/connectd/multiplex.c b/connectd/multiplex.c index d24a23f5fb63..66e9b48ca91d 100644 --- a/connectd/multiplex.c +++ b/connectd/multiplex.c @@ -407,12 +407,6 @@ static bool is_urgent(enum peer_wire type) return false; } -/* io_sock_shutdown, but in format suitable for an io_plan callback */ -static struct io_plan *io_sock_shutdown_cb(struct io_conn *conn, struct peer *unused) -{ - return io_sock_shutdown(conn); -} - static struct io_plan *encrypt_and_send(struct peer *peer, const u8 *msg TAKES, struct io_plan *(*next) @@ -447,17 +441,6 @@ static struct io_plan *encrypt_and_send(struct peer *peer, set_urgent_flag(peer, is_urgent(type)); - /* We are no longer required to do this, but we do disconnect - * after sending an error or warning. */ - if (type == WIRE_ERROR || type == WIRE_WARNING) { - /* Might already be draining... */ - if (!peer->draining) - drain_peer(peer); - - /* Close as soon as we've sent this. */ - next = io_sock_shutdown_cb; - } - /* We free this and the encrypted version in next write_to_peer */ peer->sent_to_peer = cryptomsg_encrypt_msg(peer, &peer->cs, msg); return io_write(peer->to_peer, diff --git a/tests/test_connection.py b/tests/test_connection.py index 93802c289157..a8bcfe833f4f 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -411,29 +411,18 @@ def test_opening_tiny_channel(node_factory, anchors): with pytest.raises(RpcError, match=r'They sent (ERROR|WARNING).*channel capacity is .*, which is below .*sat'): l1.fundchannel(l2, l2_min_capacity + overhead - 1) - if EXPERIMENTAL_DUAL_FUND: - assert only_one(l1.rpc.listpeers(l2.info['id'])['peers'])['connected'] - else: - wait_for(lambda: l1.rpc.listpeers(l2.info['id'])['peers'] == []) - l1.rpc.connect(l2.info['id'], 'localhost', l2.port) + assert only_one(l1.rpc.listpeers(l2.info['id'])['peers'])['connected'] + l1.fundchannel(l2, l2_min_capacity + overhead) with pytest.raises(RpcError, match=r'They sent (ERROR|WARNING).*channel capacity is .*, which is below .*sat'): l1.fundchannel(l3, l3_min_capacity + overhead - 1) - if EXPERIMENTAL_DUAL_FUND: - assert only_one(l1.rpc.listpeers(l3.info['id'])['peers'])['connected'] - else: - wait_for(lambda: l1.rpc.listpeers(l3.info['id'])['peers'] == []) - l1.rpc.connect(l3.info['id'], 'localhost', l3.port) + assert only_one(l1.rpc.listpeers(l3.info['id'])['peers'])['connected'] l1.fundchannel(l3, l3_min_capacity + overhead) with pytest.raises(RpcError, match=r'They sent (ERROR|WARNING).*channel capacity is .*, which is below .*sat'): l1.fundchannel(l4, l4_min_capacity + overhead - 1) - if EXPERIMENTAL_DUAL_FUND: - assert only_one(l1.rpc.listpeers(l4.info['id'])['peers'])['connected'] - else: - wait_for(lambda: l1.rpc.listpeers(l4.info['id'])['peers'] == []) - l1.rpc.connect(l4.info['id'], 'localhost', l4.port) + assert only_one(l1.rpc.listpeers(l4.info['id'])['peers'])['connected'] l1.fundchannel(l4, l4_min_capacity + overhead) # Note that this check applies locally too, so you can't open it if @@ -442,11 +431,7 @@ def test_opening_tiny_channel(node_factory, anchors): with pytest.raises(RpcError, match=r"channel capacity is .*, which is below .*sat"): l3.fundchannel(l2, l3_min_capacity + overhead - 1) - if EXPERIMENTAL_DUAL_FUND: - assert only_one(l3.rpc.listpeers(l2.info['id'])['peers'])['connected'] - else: - wait_for(lambda: l3.rpc.listpeers(l2.info['id'])['peers'] == []) - l3.rpc.connect(l2.info['id'], 'localhost', l2.port) + assert only_one(l3.rpc.listpeers(l2.info['id'])['peers'])['connected'] l3.fundchannel(l2, l3_min_capacity + overhead) @@ -1134,10 +1119,9 @@ def test_funding_fail(node_factory, bitcoind): with pytest.raises(RpcError, match=r'to_self_delay \d+ larger than \d+'): l1.rpc.fundchannel(l2.info['id'], int(funds / 10)) - # channels disconnect on failure (v1) - if not EXPERIMENTAL_DUAL_FUND: - wait_for(lambda: len(l1.rpc.listpeers()['peers']) == 0) - wait_for(lambda: len(l2.rpc.listpeers()['peers']) == 0) + # channels do not disconnect on failure + only_one(l1.rpc.listpeers()['peers']) + only_one(l2.rpc.listpeers()['peers']) # Restart l2 without ridiculous locktime. del l2.daemon.opts['watchtime-blocks'] @@ -1349,9 +1333,8 @@ def test_funding_external_wallet_corners(node_factory, bitcoind): l1.rpc.fundchannel_cancel(l2.info['id']) - # Cancelling causes disconnection. - wait_for(lambda: l1.rpc.listpeers(l2.info['id'])['peers'] == []) - l1.rpc.connect(l2.info['id'], 'localhost', l2.port) + # Cancelling does NOT cause disconnection. + only_one(l1.rpc.listpeers(l2.info['id'])['peers']) amount2 = 1000000 funding_addr = l1.rpc.fundchannel_start(l2.info['id'], amount2)['funding_address'] @@ -1372,8 +1355,8 @@ def test_funding_external_wallet_corners(node_factory, bitcoind): # But must unreserve inputs manually. l1.rpc.txdiscard(prep['txid']) - wait_for(lambda: l1.rpc.listpeers(l2.info['id'])['peers'] == []) - l1.rpc.connect(l2.info['id'], 'localhost', l2.port) + # Does not disconnect! + only_one(l1.rpc.listpeers(l2.info['id'])['peers']) funding_addr = l1.rpc.fundchannel_start(l2.info['id'], amount)['funding_address'] prep = l1.rpc.txprepare([{funding_addr: amount}]) @@ -2047,11 +2030,8 @@ def test_multifunding_wumbo(node_factory): with pytest.raises(RpcError, match='Amount exceeded'): l1.rpc.multifundchannel(destinations) - # Make sure it's disconnected from l2 before retrying. - if not EXPERIMENTAL_DUAL_FUND: - wait_for(lambda: l1.rpc.listpeers(l2.info['id'])['peers'] == []) - else: - assert only_one(l1.rpc.listpeers(l2.info['id'])['peers'])['connected'] + # Open failure doesn't cause disconnect + assert only_one(l1.rpc.listpeers(l2.info['id'])['peers'])['connected'] # This should succeed. destinations = [{"id": '{}@localhost:{}'.format(l2.info['id'], l2.port), diff --git a/tests/test_plugin.py b/tests/test_plugin.py index aee64ecf6bb7..d7434baff2ee 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -759,11 +759,7 @@ def test_openchannel_hook_chaining(node_factory, bitcoind): # the third plugin must now not be called anymore assert not l2.daemon.is_in_log("reject on principle") - if not EXPERIMENTAL_DUAL_FUND: - wait_for(lambda: l1.rpc.listpeers()['peers'] == []) - l1.rpc.connect(l2.info['id'], 'localhost', l2.port) - else: - assert only_one(l1.rpc.listpeers()['peers'])['connected'] + assert only_one(l1.rpc.listpeers()['peers'])['connected'] # 100000sat is good for hook_accepter, so it should fail 'on principle' # at third hook openchannel_reject.py with pytest.raises(RpcError, match=r'reject on principle'): From dd3705412d3d0b9f808b464a3a84f0e280df5833 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sun, 22 Oct 2023 14:37:31 +1030 Subject: [PATCH 06/15] common: add peer_failed_warn_nodisconnect routine for non-disconnecting warnings We generalize the current df-only "aborted" flag (and invert it) to a "disconnected" flag in the peer status message. We convert it back to the aborted flag for now inside subd.c, but that's next. Signed-off-by: Rusty Russell --- common/peer_failed.c | 27 +++++++++++++++++++++------ common/peer_failed.h | 15 +++++++++++++-- common/peer_status_wire.csv | 10 ++++++---- common/read_peer_msg.c | 2 +- lightningd/subd.c | 8 ++++---- lightningd/test/run-find_my_abspath.c | 2 +- lightningd/test/run-shuffle_fds.c | 2 +- openingd/dualopend.c | 4 ++-- 8 files changed, 49 insertions(+), 21 deletions(-) diff --git a/common/peer_failed.c b/common/peer_failed.c index 13a97887d56c..f21754e3a96e 100644 --- a/common/peer_failed.c +++ b/common/peer_failed.c @@ -25,6 +25,7 @@ peer_fatal_continue(const u8 *msg TAKES, const struct per_peer_state *pps) /* We only support one channel per peer anyway */ static void NORETURN peer_failed(struct per_peer_state *pps, + bool disconnect, bool warn, const struct channel_id *channel_id, const char *desc) @@ -40,9 +41,9 @@ peer_failed(struct per_peer_state *pps, /* Tell master the error so it can re-xmit. */ msg = towire_status_peer_error(NULL, + disconnect, desc, warn, - false, msg); peer_billboard(true, desc); peer_fatal_continue(take(msg), pps); @@ -59,7 +60,21 @@ void peer_failed_warn(struct per_peer_state *pps, desc = tal_vfmt(tmpctx, fmt, ap); va_end(ap); - peer_failed(pps, true, channel_id, desc); + peer_failed(pps, true, true, channel_id, desc); +} + +void peer_failed_warn_nodisconnect(struct per_peer_state *pps, + const struct channel_id *channel_id, + const char *fmt, ...) +{ + va_list ap; + const char *desc; + + va_start(ap, fmt); + desc = tal_vfmt(tmpctx, fmt, ap); + va_end(ap); + + peer_failed(pps, false, true, channel_id, desc); } void peer_failed_err(struct per_peer_state *pps, @@ -74,17 +89,17 @@ void peer_failed_err(struct per_peer_state *pps, desc = tal_vfmt(tmpctx, fmt, ap); va_end(ap); - peer_failed(pps, false, channel_id, desc); + peer_failed(pps, true, false, channel_id, desc); } /* We're failing because peer sent us an error/warning message */ void peer_failed_received_errmsg(struct per_peer_state *pps, - const char *desc, - bool abort_restart) + bool disconnect, + const char *desc) { u8 *msg; - msg = towire_status_peer_error(NULL, desc, false, abort_restart, NULL); + msg = towire_status_peer_error(NULL, disconnect, desc, false, NULL); peer_billboard(true, "Received %s", desc); peer_fatal_continue(take(msg), pps); } diff --git a/common/peer_failed.h b/common/peer_failed.h index c33c68d2d22a..b782e525b568 100644 --- a/common/peer_failed.h +++ b/common/peer_failed.h @@ -21,6 +21,17 @@ void peer_failed_warn(struct per_peer_state *pps, const char *fmt, ...) PRINTF_FMT(3,4) NORETURN; +/** + * peer_failed_warn_nodisconnect - Send a warning msg, don't close. + * @pps: the per-peer state. + * @channel_id: channel with error, or NULL for no particular channel. + * @fmt...: format as per status_failed(STATUS_FAIL_PEER_BAD) + */ +void peer_failed_warn_nodisconnect(struct per_peer_state *pps, + const struct channel_id *channel_id, + const char *fmt, ...) + PRINTF_FMT(3,4) NORETURN; + /** * peer_failed_err - Send a warning msg and close the channel. * @pps: the per-peer state. @@ -35,8 +46,8 @@ void peer_failed_err(struct per_peer_state *pps, /* We're failing because peer sent us an error message: NULL * channel_id means all channels. */ void peer_failed_received_errmsg(struct per_peer_state *pps, - const char *desc, - bool abort_restart) + bool disconnect, + const char *desc) NORETURN; /* I/O error */ diff --git a/common/peer_status_wire.csv b/common/peer_status_wire.csv index cc3ffdf0e8a7..66cb54a4ecff 100644 --- a/common/peer_status_wire.csv +++ b/common/peer_status_wire.csv @@ -3,11 +3,13 @@ # An error occurred: if error_for_them, that to go to them. msgtype,status_peer_error,0xFFF4 +# Do we force a disconnect from the peer? +msgdata,status_peer_error,disconnect,bool, +# The error string msgdata,status_peer_error,desc,wirestring, -# Take a deep breath, then try reconnecting to the precious little snowflake. -# (Means we sent it, since we don't hang up if they send one) +# Actually a warning, not a (fatal!) error. msgdata,status_peer_error,warning,bool, -# From an abort, no reconnect but restart daemon -msgdata,status_peer_error,abort_do_restart,bool, +# The error to send to them in future if they try to talk to us about +# this channel. msgdata,status_peer_error,len,u16, msgdata,status_peer_error,error_for_them,u8,len diff --git a/common/read_peer_msg.c b/common/read_peer_msg.c index c9a8837d26a5..d5b2a14d14fc 100644 --- a/common/read_peer_msg.c +++ b/common/read_peer_msg.c @@ -35,7 +35,7 @@ bool handle_peer_error_or_warning(struct per_peer_state *pps, err = is_peer_error(tmpctx, msg); if (err) - peer_failed_received_errmsg(pps, err, false); + peer_failed_received_errmsg(pps, true, err); /* Simply log incoming warnings */ err = is_peer_warning(tmpctx, msg); diff --git a/lightningd/subd.c b/lightningd/subd.c index f19b0ee802f5..e48f6cd7f516 100644 --- a/lightningd/subd.c +++ b/lightningd/subd.c @@ -422,18 +422,18 @@ static bool handle_peer_error(struct subd *sd, const u8 *msg, int fds[1]) struct peer_fd *peer_fd; u8 *err_for_them; bool warning; - bool aborted; + bool disconnect; if (!fromwire_status_peer_error(msg, msg, - &desc, &warning, - &aborted, &err_for_them)) + &disconnect, &desc, &warning, + &err_for_them)) return false; peer_fd = new_peer_fd_arr(msg, fds); /* Don't free sd; we may be about to free channel. */ sd->channel = NULL; - sd->errcb(channel, peer_fd, desc, warning, aborted, err_for_them); + sd->errcb(channel, peer_fd, desc, warning, !disconnect, err_for_them); return true; } diff --git a/lightningd/test/run-find_my_abspath.c b/lightningd/test/run-find_my_abspath.c index bd350e252a16..09ece1c795bc 100644 --- a/lightningd/test/run-find_my_abspath.c +++ b/lightningd/test/run-find_my_abspath.c @@ -84,7 +84,7 @@ bool fromwire_status_fail(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, enu bool fromwire_status_peer_billboard(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, bool *perm UNNEEDED, wirestring **happenings UNNEEDED) { fprintf(stderr, "fromwire_status_peer_billboard called!\n"); abort(); } /* Generated stub for fromwire_status_peer_error */ -bool fromwire_status_peer_error(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, wirestring **desc UNNEEDED, bool *warning UNNEEDED, bool *abort_do_restart UNNEEDED, u8 **error_for_them UNNEEDED) +bool fromwire_status_peer_error(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, bool *disconnect UNNEEDED, wirestring **desc UNNEEDED, bool *warning UNNEEDED, u8 **error_for_them UNNEEDED) { fprintf(stderr, "fromwire_status_peer_error called!\n"); abort(); } /* Generated stub for fromwire_status_version */ bool fromwire_status_version(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, wirestring **version UNNEEDED) diff --git a/lightningd/test/run-shuffle_fds.c b/lightningd/test/run-shuffle_fds.c index de06f7a70705..eb4e54c6f9ab 100644 --- a/lightningd/test/run-shuffle_fds.c +++ b/lightningd/test/run-shuffle_fds.c @@ -70,7 +70,7 @@ bool fromwire_status_fail(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, enu bool fromwire_status_peer_billboard(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, bool *perm UNNEEDED, wirestring **happenings UNNEEDED) { fprintf(stderr, "fromwire_status_peer_billboard called!\n"); abort(); } /* Generated stub for fromwire_status_peer_error */ -bool fromwire_status_peer_error(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, wirestring **desc UNNEEDED, bool *warning UNNEEDED, bool *abort_do_restart UNNEEDED, u8 **error_for_them UNNEEDED) +bool fromwire_status_peer_error(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, bool *disconnect UNNEEDED, wirestring **desc UNNEEDED, bool *warning UNNEEDED, u8 **error_for_them UNNEEDED) { fprintf(stderr, "fromwire_status_peer_error called!\n"); abort(); } /* Generated stub for fromwire_status_version */ bool fromwire_status_version(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, wirestring **version UNNEEDED) diff --git a/openingd/dualopend.c b/openingd/dualopend.c index 5a999c45e56a..6248ce0da3d7 100644 --- a/openingd/dualopend.c +++ b/openingd/dualopend.c @@ -333,8 +333,8 @@ static void negotiation_aborted(struct state *state, const char *why, bool abort { status_debug("aborted opening negotiation: %s", why); - /* Tell master that funding failed. */ - peer_failed_received_errmsg(state->pps, why, aborted); + /* Tell master that funding failed (don't disconnect if we aborted) */ + peer_failed_received_errmsg(state->pps, !aborted, why); } /* Softer version of 'warning' (we don't disconnect) From d2b078a3319447f27e01faa437ffb1c4568339e8 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sun, 22 Oct 2023 14:37:32 +1030 Subject: [PATCH 07/15] lightningd: pass disconnect flag to subd's errcb. Signed-off-by: Rusty Russell --- lightningd/dual_open_control.c | 14 +++++++------- lightningd/onchain_control.c | 6 +++--- lightningd/opening_common.c | 6 +++--- lightningd/opening_common.h | 6 +++--- lightningd/peer_control.c | 12 ++++++------ lightningd/peer_control.h | 6 +++--- lightningd/subd.c | 14 +++++++------- lightningd/subd.h | 14 +++++++------- 8 files changed, 39 insertions(+), 39 deletions(-) diff --git a/lightningd/dual_open_control.c b/lightningd/dual_open_control.c index 83e2abf1a14b..f5c881868eba 100644 --- a/lightningd/dual_open_control.c +++ b/lightningd/dual_open_control.c @@ -3677,9 +3677,9 @@ AUTODATA(json_command, &openchannel_abort_command); static void dualopen_errmsg(struct channel *channel, struct peer_fd *peer_fd, const char *desc, - bool warning, - bool aborted, - const u8 *err_for_them) + const u8 *err_for_them, + bool disconnect, + bool warning) { /* Clean up any in-progress open attempts */ channel_cleanup_commands(channel, desc); @@ -3694,7 +3694,7 @@ static void dualopen_errmsg(struct channel *channel, /* No peer_fd means a subd crash or disconnection. */ if (!peer_fd) { /* If the channel is unsaved, we forget it */ - channel_fail_transient(channel, true, "%s: %s", + channel_fail_transient(channel, disconnect, "%s: %s", channel->owner->name, desc); return; } @@ -3707,14 +3707,14 @@ static void dualopen_errmsg(struct channel *channel, * surprisingly, they now spew out spurious errors frequently, * and we would close the channel on them. We now support warnings * for this case. */ - if (warning || aborted) { + if (warning || !disconnect) { /* We *don't* hang up if they aborted: that's fine! */ - channel_fail_transient(channel, !aborted, "%s %s: %s", + channel_fail_transient(channel, disconnect, "%s %s: %s", channel->owner->name, warning ? "WARNING" : "ABORTED", desc); - if (aborted) { + if (!disconnect) { char *err = restart_dualopend(tmpctx, channel->peer->ld, channel, true); diff --git a/lightningd/onchain_control.c b/lightningd/onchain_control.c index 252c98427256..448b9498e83c 100644 --- a/lightningd/onchain_control.c +++ b/lightningd/onchain_control.c @@ -1496,9 +1496,9 @@ static unsigned int onchain_msg(struct subd *sd, const u8 *msg, const int *fds U static void onchain_error(struct channel *channel, struct peer_fd *pps UNUSED, const char *desc, - bool warning UNUSED, - bool aborted UNUSED, - const u8 *err_for_them UNUSED) + const u8 *err_for_them UNUSED, + bool disconnect UNUSED, + bool warning UNUSED) { channel_set_owner(channel, NULL); diff --git a/lightningd/opening_common.c b/lightningd/opening_common.c index 2ca2e6ac69b7..d9a4d92bac43 100644 --- a/lightningd/opening_common.c +++ b/lightningd/opening_common.c @@ -94,9 +94,9 @@ new_uncommitted_channel(struct peer *peer) void opend_channel_errmsg(struct uncommitted_channel *uc, struct peer_fd *peer_fd, const char *desc, - bool warning UNUSED, - bool aborted UNUSED, - const u8 *err_for_them UNUSED) + const u8 *err_for_them UNUSED, + bool disconnect UNUSED, + bool warning UNUSED) { /* Close fds, if any. */ tal_free(peer_fd); diff --git a/lightningd/opening_common.h b/lightningd/opening_common.h index f46848b98f45..8b75b56a022f 100644 --- a/lightningd/opening_common.h +++ b/lightningd/opening_common.h @@ -110,9 +110,9 @@ struct uncommitted_channel *new_uncommitted_channel(struct peer *peer); void opend_channel_errmsg(struct uncommitted_channel *uc, struct peer_fd *peer_fd, const char *desc, - bool warning UNUSED, - bool aborted UNUSED, - const u8 *err_for_them UNUSED); + const u8 *err_for_them UNUSED, + bool disconnect UNUSED, + bool warning UNUSED); void opend_channel_set_billboard(struct uncommitted_channel *uc, bool perm UNUSED, diff --git a/lightningd/peer_control.c b/lightningd/peer_control.c index 2774a570aab6..f115ae35e4aa 100644 --- a/lightningd/peer_control.c +++ b/lightningd/peer_control.c @@ -441,9 +441,9 @@ void resend_closing_transactions(struct lightningd *ld) void channel_errmsg(struct channel *channel, struct peer_fd *peer_fd, const char *desc, - bool warning, - bool aborted UNUSED, - const u8 *err_for_them) + const u8 *err_for_them, + bool disconnect, + bool warning) { /* Clean up any in-progress open attempts */ channel_cleanup_commands(channel, desc); @@ -458,7 +458,7 @@ void channel_errmsg(struct channel *channel, /* No peer_fd means a subd crash or disconnection. */ if (!peer_fd) { /* If the channel is unsaved, we forget it */ - channel_fail_transient(channel, true, "%s: %s", + channel_fail_transient(channel, disconnect, "%s: %s", channel->owner->name, desc); return; } @@ -472,7 +472,7 @@ void channel_errmsg(struct channel *channel, * would recover after a reconnect. So we downgrade, but snark * about it in the logs. */ if (!err_for_them && strends(desc, "internal error")) { - channel_fail_transient(channel, true, "%s: %s", + channel_fail_transient(channel, disconnect, "%s: %s", channel->owner->name, "lnd sent 'internal error':" " let's give it some space"); @@ -481,7 +481,7 @@ void channel_errmsg(struct channel *channel, /* This is us, sending a warning. */ if (warning) { - channel_fail_transient(channel, true, "%s sent %s", + channel_fail_transient(channel, disconnect, "%s sent %s", channel->owner->name, desc); return; diff --git a/lightningd/peer_control.h b/lightningd/peer_control.h index 32d5b2c5ae37..09d7d466b27f 100644 --- a/lightningd/peer_control.h +++ b/lightningd/peer_control.h @@ -93,9 +93,9 @@ void peer_spoke(struct lightningd *ld, const u8 *msg); void channel_errmsg(struct channel *channel, struct peer_fd *peer_fd, const char *desc, - bool warning, - bool aborted, - const u8 *err_for_them); + const u8 *err_for_them, + bool disconnect, + bool warning); u8 *p2wpkh_for_keyidx(const tal_t *ctx, struct lightningd *ld, u64 keyidx); u8 *p2tr_for_keyidx(const tal_t *ctx, struct lightningd *ld, u64 keyidx); diff --git a/lightningd/subd.c b/lightningd/subd.c index e48f6cd7f516..866edd7018c6 100644 --- a/lightningd/subd.c +++ b/lightningd/subd.c @@ -433,7 +433,7 @@ static bool handle_peer_error(struct subd *sd, const u8 *msg, int fds[1]) /* Don't free sd; we may be about to free channel. */ sd->channel = NULL; - sd->errcb(channel, peer_fd, desc, warning, !disconnect, err_for_them); + sd->errcb(channel, peer_fd, desc, err_for_them, disconnect, warning); return true; } @@ -709,9 +709,9 @@ static struct subd *new_subd(const tal_t *ctx, void (*errcb)(void *channel, struct peer_fd *peer_fd, const char *desc, - bool warning, - bool aborted, - const u8 *err_for_them), + const u8 *err_for_them, + bool disconnect, + bool warning), void (*billboardcb)(void *channel, bool perm, const char *happenings), @@ -815,9 +815,9 @@ struct subd *new_channel_subd_(const tal_t *ctx, void (*errcb)(void *channel, struct peer_fd *peer_fd, const char *desc, - bool warning, - bool aborted, - const u8 *err_for_them), + const u8 *err_for_them, + bool disconnect, + bool warning), void (*billboardcb)(void *channel, bool perm, const char *happenings), ...) diff --git a/lightningd/subd.h b/lightningd/subd.h index 587e86a2e55c..d59009d23002 100644 --- a/lightningd/subd.h +++ b/lightningd/subd.h @@ -52,9 +52,9 @@ struct subd { void (*errcb)(void *channel, struct peer_fd *peer_fd, const char *desc, - bool warning, - bool aborted, - const u8 *err_for_them); + const u8 *err_for_them, + bool disconnect, + bool warning); /* Callback to display information for listpeers RPC */ void (*billboardcb)(void *channel, bool perm, const char *happenings); @@ -134,9 +134,9 @@ struct subd *new_channel_subd_(const tal_t *ctx, void (*errcb)(void *channel, struct peer_fd *peer_fd, const char *desc, - bool warning, - bool aborted, - const u8 *err_for_them), + const u8 *err_for_them, + bool disconnect, + bool warning), void (*billboardcb)(void *channel, bool perm, const char *happenings), ...); @@ -150,7 +150,7 @@ struct subd *new_channel_subd_(const tal_t *ctx, typesafe_cb_postargs(void, void *, (errcb), \ (channel), \ struct peer_fd *, \ - const char *, bool, bool, const u8 *), \ + const char *, const u8 *, bool, bool), \ typesafe_cb_postargs(void, void *, (billboardcb), \ (channel), bool, \ const char *), \ From abe442c7fdb71a12384791f3cda7ddca6228aaed Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sun, 22 Oct 2023 14:37:32 +1030 Subject: [PATCH 08/15] pytest: revert warning delivery failure workaround Reverts 6203d250bf385cb0d59e4e8d29c2b1b40aaf43c9 "pytest: fix flake in upfront warning." now we've actually fixed the cause. Signed-off-by: Rusty Russell --- tests/test_closing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_closing.py b/tests/test_closing.py index 382233e65533..fcc62164cb24 100644 --- a/tests/test_closing.py +++ b/tests/test_closing.py @@ -3181,7 +3181,7 @@ def test_option_upfront_shutdown_script(node_factory, bitcoind, executor, chainp fut = executor.submit(l1.rpc.close, l2.info['id']) # l2 will send a warning when it dislikes shutdown script. - l2.daemon.wait_for_log(r'scriptpubkey .* is not as agreed upfront \(00143d43d226bcc27019ade52d7a3dc52a7ac1be28b8\)') + l1.daemon.wait_for_log(r'WARNING.*scriptpubkey .* is not as agreed upfront \(00143d43d226bcc27019ade52d7a3dc52a7ac1be28b8\)') # Close from l2's side and clear channel. l2.rpc.close(l1.info['id'], unilateraltimeout=1) @@ -3197,7 +3197,7 @@ def test_option_upfront_shutdown_script(node_factory, bitcoind, executor, chainp fut = executor.submit(l2.rpc.close, l1.info['id']) # l2 will send warning unilaterally when it dislikes shutdown script. - l2.daemon.wait_for_log(r'scriptpubkey .* is not as agreed upfront \(00143d43d226bcc27019ade52d7a3dc52a7ac1be28b8\)') + l1.daemon.wait_for_log(r'WARNING.*scriptpubkey .* is not as agreed upfront \(00143d43d226bcc27019ade52d7a3dc52a7ac1be28b8\)') l2.rpc.close(l1.info['id'], unilateraltimeout=1) fut.result(TIMEOUT) From afd02d94dcb51ff80563425cdd88aad06eeba0ba Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sun, 22 Oct 2023 14:37:32 +1030 Subject: [PATCH 09/15] common: move is_peer_error/is_peer_warning from read_peer_msg.c to wire_error.c Signed-off-by: Rusty Russell --- common/interactivetx.c | 2 +- common/read_peer_msg.c | 16 ---------------- common/read_peer_msg.h | 19 ------------------- common/wire_error.c | 17 +++++++++++++++++ common/wire_error.h | 19 +++++++++++++++++++ 5 files changed, 37 insertions(+), 36 deletions(-) diff --git a/common/interactivetx.c b/common/interactivetx.c index 61e3056dcfde..3442d8526537 100644 --- a/common/interactivetx.c +++ b/common/interactivetx.c @@ -18,11 +18,11 @@ #include #include #include -#include #include #include #include #include +#include /* * BOLT-f53ca2301232db780843e894f55d95d512f297f9 #2: diff --git a/common/read_peer_msg.c b/common/read_peer_msg.c index d5b2a14d14fc..b10de75e1726 100644 --- a/common/read_peer_msg.c +++ b/common/read_peer_msg.c @@ -12,22 +12,6 @@ #include #include -const char *is_peer_warning(const tal_t *ctx, const u8 *msg) -{ - if (fromwire_peektype(msg) != WIRE_WARNING) - return NULL; - /* connectd demuxes, so we only see it if channel_id is ours. */ - return sanitize_error(ctx, msg, NULL); -} - -const char *is_peer_error(const tal_t *ctx, const u8 *msg) -{ - if (fromwire_peektype(msg) != WIRE_ERROR) - return NULL; - /* connectd demuxes, so we only see it if channel_id is ours. */ - return sanitize_error(ctx, msg, NULL); -} - bool handle_peer_error_or_warning(struct per_peer_state *pps, const u8 *msg TAKES) { diff --git a/common/read_peer_msg.h b/common/read_peer_msg.h index a3d541803625..447ceb204a8d 100644 --- a/common/read_peer_msg.h +++ b/common/read_peer_msg.h @@ -8,25 +8,6 @@ struct crypto_state; struct channel_id; struct per_peer_state; -/** - * is_peer_error - if it's an error, describe it. - * @ctx: context to allocate return from. - * @msg: the peer message. - * - * If this returns non-NULL, it's usually passed to - * peer_failed_received_errmsg(). - */ -const char *is_peer_error(const tal_t *ctx, const u8 *msg); - -/** - * is_peer_warning - if it's a warning, describe it. - * @ctx: context to allocate return from. - * @msg: the peer message. - * - * If this returns non-NULL, it's usually logged. - */ -const char *is_peer_warning(const tal_t *ctx, const u8 *msg); - /** * handle_peer_error_or_warning - simple handler for errors / warnings * @pps: per-peer state. diff --git a/common/wire_error.c b/common/wire_error.c index cdda76d6685c..10a5c7fc17f2 100644 --- a/common/wire_error.c +++ b/common/wire_error.c @@ -124,3 +124,20 @@ char *sanitize_error(const tal_t *ctx, const u8 *errmsg, : type_to_string(tmpctx, struct channel_id, channel_id), (int)tal_count(data), (char *)data); } + +const char *is_peer_warning(const tal_t *ctx, const u8 *msg) +{ + if (fromwire_peektype(msg) != WIRE_WARNING) + return NULL; + /* connectd demuxes, so we only see it if channel_id is ours. */ + return sanitize_error(ctx, msg, NULL); +} + +const char *is_peer_error(const tal_t *ctx, const u8 *msg) +{ + if (fromwire_peektype(msg) != WIRE_ERROR) + return NULL; + /* connectd demuxes, so we only see it if channel_id is ours. */ + return sanitize_error(ctx, msg, NULL); +} + diff --git a/common/wire_error.h b/common/wire_error.h index b3f67e11e6a8..7797295bb878 100644 --- a/common/wire_error.h +++ b/common/wire_error.h @@ -74,4 +74,23 @@ bool channel_id_is_all(const struct channel_id *channel_id); char *sanitize_error(const tal_t *ctx, const u8 *errmsg, struct channel_id *channel_id); +/** + * is_peer_error - if it's an error, describe it. + * @ctx: context to allocate return from. + * @msg: the peer message. + * + * If this returns non-NULL, it's usually passed to + * peer_failed_received_errmsg(). + */ +const char *is_peer_error(const tal_t *ctx, const u8 *msg); + +/** + * is_peer_warning - if it's a warning, describe it. + * @ctx: context to allocate return from. + * @msg: the peer message. + * + * If this returns non-NULL, it's usually logged. + */ +const char *is_peer_warning(const tal_t *ctx, const u8 *msg); + #endif /* LIGHTNING_COMMON_WIRE_ERROR_H */ From aaa0703bc7b77f247584b62533394786364b3144 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sun, 22 Oct 2023 14:37:32 +1030 Subject: [PATCH 10/15] pyln-testing: make disconnects continue after restart. We truncate the file on stop(), but don't re-created it on start(). We didn't notice it before, but the net Signed-off-by: Rusty Russell --- contrib/pyln-testing/pyln/testing/utils.py | 21 ++++++++++++--------- tests/test_closing.py | 3 +++ tests/test_connection.py | 11 ++++++----- tests/test_pay.py | 4 ++-- 4 files changed, 23 insertions(+), 16 deletions(-) diff --git a/contrib/pyln-testing/pyln/testing/utils.py b/contrib/pyln-testing/pyln/testing/utils.py index ca1446c7e609..594e970b83e7 100644 --- a/contrib/pyln-testing/pyln/testing/utils.py +++ b/contrib/pyln-testing/pyln/testing/utils.py @@ -594,7 +594,6 @@ def __init__( self.lightning_dir = lightning_dir self.port = port self.cmd_prefix = [] - self.disconnect_file = None self.rpcproxy = bitcoindproxy self.env['CLN_PLUGIN_LOG'] = "cln_plugin=trace,cln_rpc=trace,cln_grpc=trace,debug" @@ -639,8 +638,8 @@ def __init__( def cleanup(self): # To force blackhole to exit, disconnect file must be truncated! - if self.disconnect_file: - with open(self.disconnect_file, "w") as f: + if 'dev-disconnect' in self.opts: + with open(self.opts['dev-disconnect'], "w") as f: f.truncate() @property @@ -763,12 +762,10 @@ def __init__(self, node_id, lightning_dir, bitcoind, executor, valgrind, may_fai grpc_port=self.grpc_port, ) - # If we have a disconnect string, dump it to a file for daemon. - if disconnect: - self.daemon.disconnect_file = os.path.join(lightning_dir, TEST_NETWORK, "dev_disconnect") - with open(self.daemon.disconnect_file, "w") as f: - f.write("\n".join(disconnect)) - self.daemon.opts["dev-disconnect"] = "dev_disconnect" + self.disconnect = disconnect + if self.disconnect: + self.daemon.opts["dev-disconnect"] = os.path.join(lightning_dir, TEST_NETWORK, "dev-disconnect") + # Actual population of that file occurs at start. # Various developer options let us be more aggressive self.daemon.opts["dev-fail-on-subdaemon-fail"] = None @@ -975,6 +972,12 @@ def is_synced_with_bitcoin(self, info=None): return 'warning_bitcoind_sync' not in info and 'warning_lightningd_sync' not in info def start(self, wait_for_bitcoind_sync=True, stderr_redir=False): + # If we have a disconnect string, dump it to a file for daemon. + if 'dev-disconnect' in self.daemon.opts: + with open(self.daemon.opts['dev-disconnect'], "w") as f: + if self.disconnect is not None: + f.write("\n".join(self.disconnect)) + self.daemon.start(stderr_redir=stderr_redir) # Cache `getinfo`, we'll be using it a lot self.info = self.rpc.getinfo() diff --git a/tests/test_closing.py b/tests/test_closing.py index fcc62164cb24..c96ae9f2b0f6 100644 --- a/tests/test_closing.py +++ b/tests/test_closing.py @@ -3344,6 +3344,9 @@ def test_closing_higherfee(node_factory, bitcoind, executor, anchors): # Now adjust fees so l1 asks for more on reconnect. l1.set_feerates((30000,) * 4, False) l2.set_feerates((30000,) * 4, False) + + # Allow l1 to complete next time + l1.disconnect = None l1.restart() l2.restart() l1.rpc.connect(l2.info['id'], 'localhost', l2.port) diff --git a/tests/test_connection.py b/tests/test_connection.py index a8bcfe833f4f..eb9b505f3fba 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -3120,8 +3120,9 @@ def test_dataloss_protection_no_broadcast(node_factory, bitcoind): l2.start() l1.rpc.connect(l2.info['id'], 'localhost', l2.port) - # l2 should freak out! - l2.daemon.wait_for_logs(["Peer permanent failure in CHANNELD_NORMAL: Awaiting unilateral close"]) + # l2 should freak out! But fail when trying to send error + l2.daemon.wait_for_logs(["Peer permanent failure in CHANNELD_NORMAL: Awaiting unilateral close", + 'dev_disconnect: -WIRE_ERROR']) # l1 should NOT drop to chain, since it didn't receive an error. time.sleep(5) @@ -3154,7 +3155,7 @@ def test_restart_multi_htlc_rexmit(node_factory, bitcoind, executor): # This will make it reconnect l1.stop() # Clear the disconnect so we can proceed normally - del l1.daemon.opts['dev-disconnect'] + l1.daemon.disconnect = None l1.start() # Payments will fail due to restart, but we can see results in listsendpays. @@ -3252,7 +3253,7 @@ def test_fail_unconfirmed(node_factory, bitcoind, executor): l1.daemon.kill() # Now, restart and see if it can connect OK. - del l1.daemon.opts['dev-disconnect'] + l1.daemon.disconnect = None l1.start() l1.rpc.connect(l2.info['id'], 'localhost', l2.port) @@ -3300,7 +3301,7 @@ def test_fail_unconfirmed_openchannel2(node_factory, bitcoind, executor): l1.daemon.kill() # Now, restart and see if it can connect OK. - del l1.daemon.opts['dev-disconnect'] + l1.daemon.disconnect = None l1.start() l1.rpc.connect(l2.info['id'], 'localhost', l2.port) diff --git a/tests/test_pay.py b/tests/test_pay.py index 2163db5d6fc7..ce778d27a522 100644 --- a/tests/test_pay.py +++ b/tests/test_pay.py @@ -418,7 +418,7 @@ def test_payment_success_persistence(node_factory, bitcoind, executor): # Restart l1, without disconnect stuff. del l1.daemon.opts['dev-no-reconnect'] - del l1.daemon.opts['dev-disconnect'] + l1.daemon.disconnect = None # Should reconnect, and sort the payment out. l1.start() @@ -472,7 +472,7 @@ def test_payment_failed_persistence(node_factory, executor): # Restart l1, without disconnect stuff. del l1.daemon.opts['dev-no-reconnect'] - del l1.daemon.opts['dev-disconnect'] + l1.daemon.disconnect = None # Make sure invoice has expired. time.sleep(5 + 1) From a5bd95042e2be13eaf168586bccd075998c88576 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sun, 22 Oct 2023 14:37:32 +1030 Subject: [PATCH 11/15] lightningd: close channel ourselves, if we receive an error. Previously, we would forward the message to a subd, but now we have the case where the subd is gone, but we're still connected. If the peer anything but a reestablish in that state, we drop the connection. Instead, an error should always make us fail the channel. Signed-off-by: Rusty Russell --- connectd/connectd_wire.csv | 2 ++ connectd/multiplex.c | 3 ++- lightningd/peer_control.c | 27 +++++++++++++++------ lightningd/test/run-invoice-select-inchan.c | 2 +- wallet/test/run-wallet.c | 2 +- 5 files changed, 26 insertions(+), 10 deletions(-) diff --git a/connectd/connectd_wire.csv b/connectd/connectd_wire.csv index d42d5f506a6d..afe2ac47533a 100644 --- a/connectd/connectd_wire.csv +++ b/connectd/connectd_wire.csv @@ -86,6 +86,8 @@ msgdata,connectd_peer_spoke,id,node_id, msgdata,connectd_peer_spoke,counter,u64, msgdata,connectd_peer_spoke,msgtype,u16, msgdata,connectd_peer_spoke,channel_id,channel_id, +# If msgtype == WIRE_ERROR, this is the string. +msgdata,connectd_peer_spoke,error,?wirestring, # master -> connectd: peer no longer wanted, you can disconnect. msgtype,connectd_discard_peer,2015 diff --git a/connectd/multiplex.c b/connectd/multiplex.c index 66e9b48ca91d..e31868086e36 100644 --- a/connectd/multiplex.c +++ b/connectd/multiplex.c @@ -1120,7 +1120,8 @@ static struct io_plan *read_body_from_peer_done(struct io_conn *peer_conn, take(towire_connectd_peer_spoke(NULL, &peer->id, peer->counter, t, - &channel_id))); + &channel_id, + is_peer_error(tmpctx, decrypted)))); } /* Even if we just created it, call this to catch open_channel2 */ diff --git a/lightningd/peer_control.c b/lightningd/peer_control.c index f115ae35e4aa..f9449e6f1099 100644 --- a/lightningd/peer_control.c +++ b/lightningd/peer_control.c @@ -1578,8 +1578,9 @@ void peer_spoke(struct lightningd *ld, const u8 *msg) bool dual_fund; u8 *error; int fds[2]; + char *errmsg; - if (!fromwire_connectd_peer_spoke(msg, &id, &connectd_counter, &msgtype, &channel_id)) + if (!fromwire_connectd_peer_spoke(msg, msg, &id, &connectd_counter, &msgtype, &channel_id, &errmsg)) fatal("Connectd gave bad CONNECTD_PEER_SPOKE message %s", tal_hex(msg, msg)); @@ -1596,13 +1597,25 @@ void peer_spoke(struct lightningd *ld, const u8 *msg) goto send_error; } - /* If channel is active, we raced, so ignore this: - * subd will get it soon. */ if (channel_state_wants_peercomms(channel->state)) { - log_debug(channel->log, - "channel already active"); - if (!channel->owner && - channel->state == DUALOPEND_AWAITING_LOCKIN) { + /* If they send an error, handle it immediately. */ + if (errmsg) { + channel_fail_permanent(channel, REASON_REMOTE, + "They sent %s", errmsg); + return; + } + + /* If channel is active there are two possibilities: + * 1. We have started subd, but channeld hasn't processed + * the connectd_peer_connect_subd message yet. + * 2. subd exited */ + if (channel->owner) { + /* We raced... */ + return; + } + + log_debug(channel->log, "channel already active"); + if (channel->state == DUALOPEND_AWAITING_LOCKIN) { if (socketpair(AF_LOCAL, SOCK_STREAM, 0, fds) != 0) { log_broken(ld->log, "Failed to create socketpair: %s", diff --git a/lightningd/test/run-invoice-select-inchan.c b/lightningd/test/run-invoice-select-inchan.c index 6c13a9829c2a..094b02c61419 100644 --- a/lightningd/test/run-invoice-select-inchan.c +++ b/lightningd/test/run-invoice-select-inchan.c @@ -277,7 +277,7 @@ bool fromwire_connectd_peer_connected(const tal_t *ctx UNNEEDED, const void *p U bool fromwire_connectd_peer_disconnect_done(const void *p UNNEEDED, struct node_id *id UNNEEDED, u64 *counter UNNEEDED) { fprintf(stderr, "fromwire_connectd_peer_disconnect_done called!\n"); abort(); } /* Generated stub for fromwire_connectd_peer_spoke */ -bool fromwire_connectd_peer_spoke(const void *p UNNEEDED, struct node_id *id UNNEEDED, u64 *counter UNNEEDED, u16 *msgtype UNNEEDED, struct channel_id *channel_id UNNEEDED) +bool fromwire_connectd_peer_spoke(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, struct node_id *id UNNEEDED, u64 *counter UNNEEDED, u16 *msgtype UNNEEDED, struct channel_id *channel_id UNNEEDED, wirestring **error UNNEEDED) { fprintf(stderr, "fromwire_connectd_peer_spoke called!\n"); abort(); } /* Generated stub for fromwire_dualopend_dev_memleak_reply */ bool fromwire_dualopend_dev_memleak_reply(const void *p UNNEEDED, bool *leak UNNEEDED) diff --git a/wallet/test/run-wallet.c b/wallet/test/run-wallet.c index 54190c33dd38..914c1c95eeba 100644 --- a/wallet/test/run-wallet.c +++ b/wallet/test/run-wallet.c @@ -192,7 +192,7 @@ bool fromwire_connectd_peer_connected(const tal_t *ctx UNNEEDED, const void *p U bool fromwire_connectd_peer_disconnect_done(const void *p UNNEEDED, struct node_id *id UNNEEDED, u64 *counter UNNEEDED) { fprintf(stderr, "fromwire_connectd_peer_disconnect_done called!\n"); abort(); } /* Generated stub for fromwire_connectd_peer_spoke */ -bool fromwire_connectd_peer_spoke(const void *p UNNEEDED, struct node_id *id UNNEEDED, u64 *counter UNNEEDED, u16 *msgtype UNNEEDED, struct channel_id *channel_id UNNEEDED) +bool fromwire_connectd_peer_spoke(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, struct node_id *id UNNEEDED, u64 *counter UNNEEDED, u16 *msgtype UNNEEDED, struct channel_id *channel_id UNNEEDED, wirestring **error UNNEEDED) { fprintf(stderr, "fromwire_connectd_peer_spoke called!\n"); abort(); } /* Generated stub for fromwire_dualopend_dev_memleak_reply */ bool fromwire_dualopend_dev_memleak_reply(const void *p UNNEEDED, bool *leak UNNEEDED) From 863f8fb07f17ff9b0a7574b62ab87cad7b761f32 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sun, 22 Oct 2023 14:37:32 +1030 Subject: [PATCH 12/15] channeld: don't hang up immediately after sending bad reestablish warning. This gives the peer a chance to send an error, which will make us drop to chain. Signed-off-by: Rusty Russell Fixes: #5818 --- channeld/channeld.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/channeld/channeld.c b/channeld/channeld.c index 3efdd4e27f3d..cc3701eb7c95 100644 --- a/channeld/channeld.c +++ b/channeld/channeld.c @@ -4841,13 +4841,17 @@ static void peer_reconnect(struct peer *peer, retransmit_revoke_and_ack = true; } else if (next_revocation_number < peer->next_index[LOCAL] - 1) { /* Send a warning here! Because this is what it looks like if peer is - * in the past, and they might still recover. */ - peer_failed_warn(peer->pps, - &peer->channel_id, - "bad reestablish revocation_number: %"PRIu64 - " vs %"PRIu64, - next_revocation_number, - peer->next_index[LOCAL]); + * in the past, and they might still recover. + * + * We don't disconnect: they might send an error, meaning + * we will force-close the channel for them. + */ + peer_failed_warn_nodisconnect(peer->pps, + &peer->channel_id, + "bad reestablish revocation_number: %"PRIu64 + " vs %"PRIu64, + next_revocation_number, + peer->next_index[LOCAL]); } else if (next_revocation_number > peer->next_index[LOCAL] - 1) { if (!check_extra_fields) /* They don't support option_data_loss_protect or From e9b6423c9da52802eec888859dfd01e212dbadc5 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sun, 22 Oct 2023 14:37:32 +1030 Subject: [PATCH 13/15] pytest: be sure that we receive error on datalose failure. It was intermittant before: I added a sleep(1) in the code before sending the error (temporarily) to make it always triggers. Signed-off-by: Rusty Russell --- channeld/channeld.c | 1 + tests/test_connection.py | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/channeld/channeld.c b/channeld/channeld.c index cc3701eb7c95..ff9d183afa93 100644 --- a/channeld/channeld.c +++ b/channeld/channeld.c @@ -4437,6 +4437,7 @@ static void check_future_dataloss_fields(struct peer *peer, take(towire_channeld_fail_fallen_behind(NULL, remote_current_per_commitment_point))); + sleep(1); /* We have to send them an error to trigger dropping to chain. */ peer_failed_err(peer->pps, &peer->channel_id, "Awaiting unilateral close"); diff --git a/tests/test_connection.py b/tests/test_connection.py index eb9b505f3fba..cce0c50005dd 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -3055,9 +3055,8 @@ def test_dataloss_protection(node_factory, bitcoind): assert not l2.daemon.is_in_log('sendrawtx exit 0', start=l2.daemon.logsearch_start) - # l1 should drop to chain, but doesn't always receive ERROR before it sends warning. - # We have to reconnect once - l1.rpc.connect(l2.info['id'], 'localhost', l2.port) + # l1 should receive error and drop to chain + l1.daemon.wait_for_log("They sent ERROR.*Awaiting unilateral close") l1.wait_for_channel_onchain(l2.info['id']) closetxid = only_one(bitcoind.rpc.getrawmempool(False)) From b85431bfef315870041689e202f73dcb4494781e Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sun, 22 Oct 2023 14:37:32 +1030 Subject: [PATCH 14/15] lightningd: let channeld/dual_openingd send error to peer. We do it here, but it's not necessary, and we also deprive them of the chance to do so (since we kill them). Signed-off-by: Rusty Russell --- lightningd/channel_control.c | 5 ----- tests/test_connection.py | 4 ++-- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/lightningd/channel_control.c b/lightningd/channel_control.c index 6725e43048f7..bfae7643e2aa 100644 --- a/lightningd/channel_control.c +++ b/lightningd/channel_control.c @@ -1059,11 +1059,6 @@ void channel_fallen_behind(struct channel *channel, const u8 *msg) fatal("Our own id invalid?"); channel->future_per_commitment_point = any; } - - /* Peer sees this, so send a generic msg about unilateral close. */ - channel_fail_permanent(channel, - REASON_LOCAL, - "Awaiting unilateral close"); } static void diff --git a/tests/test_connection.py b/tests/test_connection.py index cce0c50005dd..a884301e77d1 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -3048,7 +3048,7 @@ def test_dataloss_protection(node_factory, bitcoind): l2.start() # l2 should freak out! - l2.daemon.wait_for_log("Peer permanent failure in CHANNELD_NORMAL: Awaiting unilateral close") + l2.daemon.wait_for_log("Peer permanent failure in CHANNELD_NORMAL:.*Awaiting unilateral close") # l2 must NOT drop to chain. l2.daemon.wait_for_log("Cannot broadcast our commitment tx: they have a future one") @@ -3120,7 +3120,7 @@ def test_dataloss_protection_no_broadcast(node_factory, bitcoind): l1.rpc.connect(l2.info['id'], 'localhost', l2.port) # l2 should freak out! But fail when trying to send error - l2.daemon.wait_for_logs(["Peer permanent failure in CHANNELD_NORMAL: Awaiting unilateral close", + l2.daemon.wait_for_logs(["Peer permanent failure in CHANNELD_NORMAL:.* Awaiting unilateral close", 'dev_disconnect: -WIRE_ERROR']) # l1 should NOT drop to chain, since it didn't receive an error. From 2c047d2e84bc6a91ead9226ae8a405b8d356540b Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 23 Oct 2023 11:35:39 +1030 Subject: [PATCH 15/15] connectd: use shutdown() not close() on TCP sockets for dev-disconnect. close() is allowed to lose data, and I saw this in CI: ``` 2023-10-22T05:12:36.6576005Z ____________________________ test_permfail_htlc_out ____________________________ 2023-10-22T05:12:36.6608511Z [gw2] linux -- Python 3.8.18 /home/runner/.cache/pypoetry/virtualenvs/cln-meta-project-AqJ9wMix-py3.8/bin/python 2023-10-22T05:12:36.6611663Z 2023-10-22T05:12:36.6614768Z node_factory = 2023-10-22T05:12:36.6623694Z bitcoind = 2023-10-22T05:12:36.6627092Z executor = 2023-10-22T05:12:36.6627701Z 2023-10-22T05:12:36.6628051Z def test_permfail_htlc_out(node_factory, bitcoind, executor): 2023-10-22T05:12:36.6631192Z # Test case where we fail with unsettled outgoing HTLC. 2023-10-22T05:12:36.6634154Z disconnects = ['+WIRE_REVOKE_AND_ACK', 'permfail'] 2023-10-22T05:12:36.6635106Z l1 = node_factory.get_node(options={'dev-no-reconnect': None}) 2023-10-22T05:12:36.6637321Z # Feerates identical so we don't get gratuitous commit to update them 2023-10-22T05:12:36.6642691Z l2 = node_factory.get_node(disconnect=disconnects, 2023-10-22T05:12:36.6644734Z feerates=(7500, 7500, 7500, 7500)) 2023-10-22T05:12:36.6647205Z 2023-10-22T05:12:36.6649671Z l1.rpc.connect(l2.info['id'], 'localhost', l2.port) 2023-10-22T05:12:36.6650460Z l2.daemon.wait_for_log('Handed peer, entering loop') 2023-10-22T05:12:36.6654865Z l2.fundchannel(l1, 10**6) 2023-10-22T05:12:36.6655305Z 2023-10-22T05:12:36.6657810Z # This will fail at l2's end. 2023-10-22T05:12:36.6660554Z t = executor.submit(l2.pay, l1, 200000000) 2023-10-22T05:12:36.6662947Z 2023-10-22T05:12:36.6665147Z l2.daemon.wait_for_log('dev_disconnect permfail') 2023-10-22T05:12:36.6668530Z l2.wait_for_channel_onchain(l1.info['id']) 2023-10-22T05:12:36.6671588Z bitcoind.generate_block(1) 2023-10-22T05:12:36.6674510Z > l1.daemon.wait_for_log('Their unilateral tx, old commit point') 2023-10-22T05:12:36.6675001Z 2023-10-22T05:12:36.6675212Z tests/test_closing.py:3027: ... 2023-10-22T05:12:36.8784390Z lightningd-2 2023-10-22T04:41:04.448Z DEBUG 0266e4598d1d3c415f572a8488830b60f7e744ed9235eb0b1ba93283b315c03518-connectd: dev_disconnect: +WIRE_REVOKE_AND_ACK (WIRE_REVOKE_AND_ACK) 2023-10-22T05:12:36.8786260Z lightningd-2 2023-10-22T04:41:04.452Z INFO 0266e4598d1d3c415f572a8488830b60f7e744ed9235eb0b1ba93283b315c03518-channeld-chan#1: Peer connection lost 2023-10-22T05:12:36.8788076Z lightningd-2 2023-10-22T04:41:04.453Z DEBUG 0266e4598d1d3c415f572a8488830b60f7e744ed9235eb0b1ba93283b315c03518-channeld-chan#1: Status closed, but not exited. Killing 2023-10-22T05:12:36.8789915Z lightningd-1 2023-10-22T04:41:04.454Z INFO 022d223620a359a47ff7f7ac447c85c46c923da53389221a0054c11c1e3ca31d59-channeld-chan#1: Peer connection lost ``` Note that l1 doesn't receive WIRE_REVOKE_AND_ACK! Signed-off-by: Rusty Russell --- connectd/multiplex.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/connectd/multiplex.c b/connectd/multiplex.c index e31868086e36..0e80eedda351 100644 --- a/connectd/multiplex.c +++ b/connectd/multiplex.c @@ -407,6 +407,12 @@ static bool is_urgent(enum peer_wire type) return false; } +/* io_sock_shutdown, but in format suitable for an io_plan callback */ +static struct io_plan *io_sock_shutdown_cb(struct io_conn *conn, struct peer *unused) +{ + return io_sock_shutdown(conn); +} + static struct io_plan *encrypt_and_send(struct peer *peer, const u8 *msg TAKES, struct io_plan *(*next) @@ -423,7 +429,8 @@ static struct io_plan *encrypt_and_send(struct peer *peer, case DEV_DISCONNECT_AFTER: /* Disallow reads from now on */ peer->dev_read_enabled = false; - next = (void *)io_close_cb; + /* Using io_close here can lose the data we're about to send! */ + next = io_sock_shutdown_cb; break; case DEV_DISCONNECT_BLACKHOLE: /* Disable both reads and writes from now on */