Wait forever for call to respond

It takes more than 5s on some systems at the moment - and this then dumps an unhelpful crashdump to the user.  Make a longer default timeout, allow the timeout to be passed by the operator, and also return more operator-friendly error on timeout occurring.

src/riak_kv_http_cache.erl

@@ -1,7 +1,7 @@
 -module(riak_kv_http_cache).
 
 -export([start_link/0,
-	 get_stats/0]).
+	 get_stats/1]).
 
 -export([init/1,
 	 handle_call/3,
@@ -10,20 +10,19 @@
 	 terminate/2,
 	 code_change/3]).
 
--define(SERVER, ?MODULE).
-
 -record(st,
     {
         ts :: undefined|erlang:timestamp(),
         stats = []
     }
 ).
 
+
 start_link() ->
     gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
 
-get_stats() ->
-    gen_server:call(?MODULE, get_stats).
+get_stats(Timeout) ->
+    gen_server:call(?MODULE, get_stats, Timeout).
 
 init(_) ->
     {ok, #st{}}.
@@ -45,7 +44,7 @@ code_change(_, S, _) ->
     {ok, S}.
 
 check_cache(#st{ts = undefined} = S) ->
-    Stats = do_get_stats(),
+    Stats = riak_kv_status:get_stats(web),
     S#st{ts = os:timestamp(), stats = Stats};
 check_cache(#st{ts = Then} = S) ->
     CacheTime = application:get_env(riak_kv, http_stats_cache_seconds, 1),
@@ -54,9 +53,6 @@ check_cache(#st{ts = Then} = S) ->
         true ->
             S;
         false ->
-            Stats = do_get_stats(),
+            Stats = riak_kv_status:get_stats(web),
             S#st{ts = os:timestamp(), stats = Stats}
-    end.
-
-do_get_stats() ->
-    riak_kv_wm_stats:get_stats().
+    end.

src/riak_kv_stat.erl

@@ -26,11 +26,6 @@
 %%      Update each stat with the exported function update/1. Add
 %%      a new stat to the internal stats/0 func to register a new stat with
 %%      folsom.
-%%
-%%      Get the latest aggregation of stats with the exported function
-%%      get_stats/0. Or use folsom_metrics:get_metric_value/1,
-%%      or riak_core_stat_q:get_stats/1.
-%%
 
 -module(riak_kv_stat).
 
@@ -41,7 +36,7 @@
 -endif.
 
 %% API
--export([start_link/0, get_stats/0,
+-export([start_link/0,
          update/1, perform_update/1, register_stats/0, unregister_vnode_stats/1, produce_stats/0,
          leveldb_read_block_errors/0, stat_update_error/3, stop/0]).
 -export([track_bucket/1, untrack_bucket/1]).
@@ -71,11 +66,6 @@ unregister_vnode_stats(Index) ->
     unregister_per_index(heads, Index),
     unregister_per_index(puts, Index).
 
-%% @spec get_stats() -> proplist()
-%% @doc Get the current aggregation of stats.
-get_stats() ->
-    riak_kv_wm_stats:get_stats().
-
 
 %% Creation of a dynamic stat _must_ be serialized.
 register_stat(Name, Type) ->
@@ -1163,7 +1153,7 @@ create_or_update_histogram_test() ->
         Metric = [riak_kv,put_fsm,counter,time],
         ok = repeat_create_or_update(Metric, 1, histogram, 100),
         ?assertNotEqual(exometer:get_value(Metric), 0),
-        Stats = get_stats(),
+        Stats = riak_kv_status:get_stats(web),
         ?LOG_INFO("stats prop list ~s", [Stats]),
         ?assertNotEqual(proplists:get_value({node_put_fsm_counter_time_mean}, Stats), 0)
     after

src/riak_kv_wm_stats.erl

@@ -29,14 +29,17 @@
          content_types_provided/2,
          service_available/2,
          forbidden/2,
+         malformed_request/2,
          produce_body/2,
          pretty_print/2
         ]).
--export([get_stats/0]).
+
+-define(TIMEOUT, 30000).
 
 -include_lib("webmachine/include/webmachine.hrl").
+-include("riak_kv_wm_raw.hrl").
 
--record(ctx, {}).
+-record(ctx, {timeout = ?TIMEOUT :: non_neg_integer()}).
 
 init(_) ->
     {ok, #ctx{}}.
@@ -66,25 +69,67 @@ content_types_provided(ReqData, Context) ->
       {"text/plain", pretty_print}],
      ReqData, Context}.
 
-
 service_available(ReqData, Ctx) ->
     {true, ReqData, Ctx}.
 
+malformed_request(RD, Ctx) ->
+    case wrq:get_qs_value("timeout", RD) of
+        undefined ->
+            {false, RD, Ctx};
+        TimeoutStr ->
+            try
+                case list_to_integer(TimeoutStr) of
+                    Timeout when Timeout > 0 ->
+                        {false, RD, Ctx#ctx{timeout=Timeout}}
+                end
+            catch
+                _:_ ->
+                    {true,
+                        wrq:append_to_resp_body(
+                            io_lib:format(
+                                "Bad timeout value ~0p",
+                                [TimeoutStr]
+                            ),
+                        wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD)),
+                        Ctx}
+            end
+    end.
+
 forbidden(RD, Ctx) ->
     {riak_kv_wm_utils:is_forbidden(RD), RD, Ctx}.
 
-produce_body(ReqData, Ctx) ->
-    Stats= riak_kv_http_cache:get_stats(),
-    Body = mochijson2:encode({struct, Stats}),
-    {Body, ReqData, Ctx}.
+produce_body(RD, Ctx) ->
+    try 
+        Stats = riak_kv_http_cache:get_stats(Ctx#ctx.timeout),
+        Body = mochijson2:encode({struct, Stats}),
+        {Body, RD, Ctx}
+    catch
+        exit:{timeout, _} ->
+            {
+                {halt, 503},
+                wrq:set_resp_header(
+                    ?HEAD_CTYPE,
+                    "text/plain",
+                    wrq:append_to_response_body(
+                        io_lib:format(
+                            "Request timed out after ~w ms",
+                            [Ctx#ctx.timeout]
+                        ),
+                        RD
+                    )
+                ),
+                Ctx
+            }
+    end.
 
 %% @spec pretty_print(webmachine:wrq(), context()) ->
 %%          {string(), webmachine:wrq(), context()}
 %% @doc Format the respons JSON object is a "pretty-printed" style.
-pretty_print(RD1, C1=#ctx{}) ->
-    {Json, RD2, C2} = produce_body(RD1, C1),
-    {json_pp:print(binary_to_list(list_to_binary(Json))), RD2, C2}.
-
+pretty_print(RD, Ctx) ->
+    case produce_body(RD, Ctx) of
+        {{halt, RepsonseCode}, UpdRD, UpdCtx} ->
+            {{halt, RepsonseCode}, UpdRD, UpdCtx};
+        {Json, UpdRD, UpdCtx} ->
+            {json_pp:print(Json), UpdRD, UpdCtx}
+    end.
 
-get_stats() ->
-    riak_kv_status:get_stats(web).