From f2a8c7eae21312105e5170ae90cd6b7617effc90 Mon Sep 17 00:00:00 2001
From: Rusty Russell <rusty@rustcorp.com.au>
Date: Fri, 10 Nov 2023 13:21:38 +1030
Subject: [PATCH] clnrest: don't convert *_msat fields to strings.

We have a global JSON encoder hack, which means that any field ending in msat gets special treatment (so we can safely talk to lightningd, even if a field expects satoshi amounts, we are explicit).  However, requests uses the JSON parser and DOES NOT want this conversion when sending it out as an HTTP response!

The simplest local fix we could find was Shahana's suggestion to iterate and covert away from Millisatoshi(): the reverse of what our JSON encoder does.

Fixes: https://github.com/ElementsProject/lightning/issues/6848
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
---
 plugins/clnrest/utilities/shared.py | 24 +++++++++++++++++++++++-
 tests/test_clnrest.py               | 23 +++++++++++++++++++++++
 2 files changed, 46 insertions(+), 1 deletion(-)

diff --git a/plugins/clnrest/utilities/shared.py b/plugins/clnrest/utilities/shared.py
index 509ec83627b6..2c71093c48b3 100644
--- a/plugins/clnrest/utilities/shared.py
+++ b/plugins/clnrest/utilities/shared.py
@@ -1,5 +1,6 @@
 import json5
 import ipaddress
+import pyln.client
 
 
 CERTS_PATH, REST_PROTOCOL, REST_HOST, REST_PORT, REST_CSP, REST_CORS_ORIGINS = "", "", "", "", "", []
@@ -58,8 +59,29 @@ def set_config(options):
     return None
 
 
+def convert_millisatoshis(item):
+    """
+    The global JSON encoder has been replaced (see
+    monkey_patch_json!)  by one that turns Millisatoshi class object
+    into strings ending in msat.  We do not want the http response
+    to be encoded like that!  pyln-client should probably not do that,
+    but meanwhile, convert them to integers.
+    """
+    if isinstance(item, dict):
+        ret = {}
+        for k in item:
+            ret[k] = convert_millisatoshis(item[k])
+    elif isinstance(item, list):
+        ret = [convert_millisatoshis(i) for i in item]
+    elif isinstance(item, pyln.client.Millisatoshi):
+        ret = int(item)
+    else:
+        ret = item
+    return ret
+
+
 def call_rpc_method(plugin, rpc_method, payload):
-    return plugin.rpc.call(rpc_method, payload)
+    return convert_millisatoshis(plugin.rpc.call(rpc_method, payload))
 
 
 def verify_rune(plugin, rune, rpc_method, rpc_params):
diff --git a/tests/test_clnrest.py b/tests/test_clnrest.py
index 0fae123a4bf8..fa019d16b97c 100644
--- a/tests/test_clnrest.py
+++ b/tests/test_clnrest.py
@@ -1,6 +1,7 @@
 from ephemeral_port_reserve import reserve
 from fixtures import *  # noqa: F401,F403
 from pyln.testing.utils import env, TEST_NETWORK
+from pyln.client import Millisatoshi
 import unittest
 import requests
 from pathlib import Path
@@ -236,6 +237,28 @@ def test_clnrest_rpc_method(node_factory):
     assert 'bolt11' in response.json()
 
 
+def test_clnrest_large_response(node_factory):
+    """Test a large reply still works (and msat fields are integers!)"""
+    # start a node with clnrest
+    l1, base_url, ca_cert = start_node_with_clnrest(node_factory)
+    http_session = http_session_with_retry()
+
+    # Add 500 invoices, test list
+    NUM_INVOICES = 500
+    for i in range(NUM_INVOICES):
+        l1.rpc.invoice(amount_msat=100, label=str(i), description="inv")
+
+    rune = l1.rpc.createrune()['rune']
+    response = http_session.post(base_url + '/v1/listinvoices', headers={'Rune': rune},
+                                 verify=ca_cert)
+    # No, this doesn't return JSON, it *parses* it into a Python object!
+    resp = response.json()
+
+    # Make sure it hasn't turned msat fields into strings!
+    assert not isinstance(resp['invoices'][0]['amount_msat'], Millisatoshi)
+    assert len(resp['invoices']) == NUM_INVOICES
+
+
 # Tests for websocket are written separately to avoid flake8
 # to complain with the errors F811 like this "F811 redefinition of
 # unused 'message'".