diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eaff648..2d49263 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,4 +15,4 @@ jobs: with: otp-version: ${{matrix.otp}} rebar3-version: ${{matrix.rebar3}} - - run: sudo apt update && sudo apt install -y make gcc libevent-dev libcurl4-openssl-dev libssl-dev && rebar3 update && rebar3 ct && rebar3 dialyzer + - run: sudo apt update && sudo apt install -y make gcc libevent-dev libcurl4-openssl-dev libssl-dev && rebar3 update && rebar3 ct && rebar3 dialyzer && rebar3 lint diff --git a/.gitignore b/.gitignore index c705980..53694eb 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ _build .vagrant/ local/ /rebar3.crashdump +doc \ No newline at end of file diff --git a/Dockerfile.katipo_build b/Dockerfile.katipo_build deleted file mode 100644 index 44d2d15..0000000 --- a/Dockerfile.katipo_build +++ /dev/null @@ -1,24 +0,0 @@ -FROM ubuntu:19.04 - -MAINTAINER Paul Oliver - -RUN apt-get -qq update \ - && apt-get -qq -y install wget gnupg \ - && wget https://packages.erlang-solutions.com/erlang-solutions_1.0_all.deb \ - && dpkg -i erlang-solutions_1.0_all.deb \ - && rm erlang-solutions_1.0_all.deb - -RUN apt-get -qq update \ - && DEBIAN_FRONTEND=noninteractive \ - apt-get -qq -y install \ - libevent-dev \ - libcurl4-openssl-dev \ - erlang \ - make \ - curl \ - libssl-dev \ - gcc \ - docker \ - && rm -rf /var/lib/apt/lists/* \ - && curl --location https://github.com/erlang/rebar3/releases/download/3.10.0/rebar3 > /usr/local/bin/rebar3 \ - && chmod 755 /usr/local/bin/rebar3 diff --git a/README.md b/README.md index f4eaf11..58d9fb5 100644 --- a/README.md +++ b/README.md @@ -49,27 +49,6 @@ Req = #{url => <<"https://example.com">>. body := RespBody}} = katipo:req(Pool, Req). ``` -Session interface. Cookies handled automatically and options merged. Inspired by [Requests sessions](http://docs.python-requests.org/en/latest/user/advanced/#session-objects). - -```erlang -{ok, _} = application:ensure_all_started(katipo). -Pool = api_server, -{ok, _} = katipo_pool:start(Pool, 2, [{pipelining, multiplex}]). -ReqHeaders = [{<<"User-Agent">>, <<"katipo">>}]. -Opts = #{url => <<"https://example.com">>. - method => post, - headers => ReqHeaders, - connecttimeout_ms => 5000, - proxy => <<"http://127.0.0.1:9000">>, - ssl_verifyhost => false, - ssl_verifypeer => false}. -{ok, Session} = katipo_session:new(Pool, Opts). -{{ok, #{status := 200}}, Session2} = - katipo_session:req(#{body => <<"some data">>}, Session). -{{ok, #{status := 200}}, Session3} = - katipo_session:req(#{body => <<"different payload data">>}, Session2). -``` - ### Why We wanted a compatible and high-performance HTTP client so took @@ -124,7 +103,7 @@ katipo:Method(Pool :: atom(), URL :: binary(), ReqOptions :: map()). | `sslkey` | `binary()` | `undefined` | [docs](https://curl.haxx.se/libcurl/c/CURLOPT_SSLKEY.html) | | `sslkey_blob` | `binary()` (DER format) | `undefined` | [docs](https://curl.haxx.se/libcurl/c/CURLOPT_SSLKEY_BLOB.html) curl >= 7.71.0 | | `keypasswd` | `binary()` | `undefined` | [docs](https://curl.haxx.se/libcurl/c/CURLOPT_KEYPASSWD.html) | -| `http_auth` | `basic | digest | ntlm | negotiate` | `undefined` | [docs](https://curl.haxx.se/libcurl/c/CURLOPT_HTTPAUTH.html) | +| `http_auth` | `basic`
`digest`
`ntlm`
`negotiate` | `undefined` | [docs](https://curl.haxx.se/libcurl/c/CURLOPT_HTTPAUTH.html) | | `userpwd` | `binary()` | `undefined` | [docs](https://curl.haxx.se/libcurl/c/CURLOPT_USERPWD.html) | #### Responses @@ -143,7 +122,7 @@ katipo:Method(Pool :: atom(), URL :: binary(), ReqOptions :: map()). | Option | Type | Default | Note | |:------------------------|:------------------------------|:-------------|:-----------------------------------------------------------------------------------------------| -| `pipelining` | `nothing | http1 | multiplex` | `nothing` | HTTP pipelining [CURLMOPT_PIPELINING](https://curl.haxx.se/libcurl/c/CURLMOPT_PIPELINING.html) | +| `pipelining` | `nothing`
`http1`
`multiplex` | `nothing` | HTTP pipelining [CURLMOPT_PIPELINING](https://curl.haxx.se/libcurl/c/CURLMOPT_PIPELINING.html) | | `max_pipeline_length` | `non_neg_integer()` | 100 | | | `max_total_connections` | `non_neg_integer()` | 0 (no limit) | [docs](https://curl.haxx.se/libcurl/c/CURLMOPT_MAX_TOTAL_CONNECTIONS.html) | @@ -161,37 +140,20 @@ katipo:Method(Pool :: atom(), URL :: binary(), ReqOptions :: map()). * redirect_time * starttransfer_time -### Dependencies +### System dependencies -#### Ubuntu Trusty +* libevent-dev +* libcurl4-openssl-dev +* make +* curl +* libssl-dev +* gcc -```sh -sudo apt-get install git libwxgtk2.8-0 libwxbase2.8-0 libevent-dev libcurl4-openssl-dev libcurl4-openssl-dev +## Testing -wget http://packages.erlang-solutions.com/site/esl/esl-erlang/FLAVOUR_1_esl/esl-erlang_18.0-1~ubuntu~trusty_amd64.deb - -sudo dpkg -i esl-erlang_18.0-1~ubuntu~trusty_amd64.deb -``` -#### Fedora - -```sh -sudo dnf install libevent.x86_64 libcurl.x86_64 libevent-devel.x86_64 -``` - -#### OSX - -```sh -brew install --with-c-ares --with-nghttp2 curl -brew install libevent -``` - -### Building - -```sh -rebar3 compile -``` +The official Erlang Docker [image](https://hub.docker.com/_/erlang) +has everything needed to build and test Katipo ### TODO * A more structured way to ifdef features based on curl version -* Better session interface diff --git a/elvis.config b/elvis.config new file mode 100644 index 0000000..b26e2d6 --- /dev/null +++ b/elvis.config @@ -0,0 +1,23 @@ +[{elvis, [ + {config, [ + #{ dirs => ["src/**"] + , filter => "*.erl" + , ruleset => erl_files + , rules => [{elvis_style, atom_naming_convention, #{ regex => "^([a-z][a-z0-9]*_?)+([a-z0-9_]*)$", enclosed_atoms => ".*"}} + , {elvis_style, god_modules, #{limit => 30}}] + } + , #{ dirs => ["c_src/**"] + , filter => "Makefile" + , ruleset => makefiles + , rules => [] } + , #{ dirs => ["."] + , filter => "rebar.config" + , ruleset => rebar_config + , rules => [] } + , #{ dirs => ["."] + , filter => "elvis.config" + , ruleset => elvis_config + , rules => [] } + ]} + , {verbose, true} +]}]. diff --git a/rebar.config b/rebar.config index c1e3d1a..3a8477e 100644 --- a/rebar.config +++ b/rebar.config @@ -13,11 +13,10 @@ [ {test, [{deps, - [{jsx, "2.9.0"}, - {meck, "0.8.10"}, + [{jsx, "3.1.0"}, + {meck, "0.9.2"}, {cowboy, "2.9.0"}, - {ephemeral, "2.0.4"}, - {proper, "1.3.0"} + {ephemeral, "2.0.4"} ]}] }] }. @@ -40,7 +39,7 @@ deprecated_functions]}. {plugins, [rebar3_hex, - rebar3_proper, + rebar3_lint, {coveralls, "1.4.0"}]}. {cover_enabled, true}. {cover_export_enabled, true}. diff --git a/src/katipo.app.src b/src/katipo.app.src index 6af2f09..cba59fc 100644 --- a/src/katipo.app.src +++ b/src/katipo.app.src @@ -1,6 +1,6 @@ {application, 'katipo', [{description, "HTTP client based on libcurl"}, - {vsn, "1.0.4"}, + {vsn, "1.1.0"}, {registered, []}, {mod, {'katipo_app', []}}, {applications, @@ -24,6 +24,5 @@ "src/katipo_cow_qs.erl", "src/katipo_metrics.erl", "src/katipo_pool.erl", - "src/katipo_session.erl", "src/katipo_sup.erl"]} ]}. diff --git a/src/katipo.erl b/src/katipo.erl index a1351c4..2650f7f 100644 --- a/src/katipo.erl +++ b/src/katipo.erl @@ -2,7 +2,7 @@ -behaviour(gen_server). --compile({no_auto_import,[put/2]}). +-compile({no_auto_import, [put/2]}). -export([start_link/1]). @@ -12,7 +12,6 @@ -export([handle_info/2]). -export([terminate/2]). -export([code_change/3]). - -export([req/2]). -export([get/2]). -export([get/3]). @@ -68,38 +67,38 @@ -record(state, {port :: port(), reqs = #{} :: map()}). --define(get, 0). --define(post, 1). --define(put, 2). --define(head, 3). --define(options, 4). --define(patch, 5). --define(delete, 6). - --define(connecttimeout_ms, 5). --define(followlocation, 6). --define(ssl_verifyhost, 7). --define(timeout_ms, 8). --define(maxredirs, 9). --define(ssl_verifypeer, 10). --define(capath, 11). --define(http_auth, 12). --define(username, 13). --define(password, 14). --define(proxy, 15). --define(cacert, 16). --define(tcp_fastopen, 17). --define(interface, 18). --define(unix_socket_path, 19). --define(lock_data_ssl_session, 20). --define(doh_url, 21). --define(http_version, 22). --define(verbose, 23). --define(sslcert, 24). --define(sslkey, 25). --define(sslkey_blob, 26). --define(keypasswd, 27). --define(userpwd, 28). +-define(GET, 0). +-define(POST, 1). +-define(PUT, 2). +-define(HEAD, 3). +-define(OPTIONS, 4). +-define(PATCH, 5). +-define(DELETE, 6). + +-define(CONNECTTIMEOUT_MS, 5). +-define(FOLLOWLOCATION, 6). +-define(SSL_VERIFYHOST, 7). +-define(TIMEOUT_MS, 8). +-define(MAXREDIRS, 9). +-define(SSL_VERIFYPEER, 10). +-define(CAPATH, 11). +-define(HTTP_AUTH, 12). +-define(USERNAME, 13). +-define(PASSWORD, 14). +-define(PROXY, 15). +-define(CACERT, 16). +-define(TCP_FASTOPEN, 17). +-define(INTERFACE, 18). +-define(UNIX_SOCKET_PATH, 19). +-define(LOCK_DATA_SSL_SESSION, 20). +-define(DOH_URL, 21). +-define(HTTP_VERSION, 22). +-define(VERBOSE, 23). +-define(SSLCERT, 24). +-define(SSLKEY, 25). +-define(SSLKEY_BLOB, 26). +-define(KEYPASSWD, 27). +-define(USERPWD, 28). -define(DEFAULT_REQ_TIMEOUT, 30000). -define(FOLLOWLOCATION_TRUE, 1). @@ -123,7 +122,7 @@ -define(METHODS, [get, post, put, head, options, patch, delete]). -type method() :: get | post | put | head | options | patch | delete. --type method_int() :: ?get | ?post | ?put | ?head | ?options | ?patch | ?delete. +-type method_int() :: ?GET | ?POST | ?PUT | ?HEAD | ?OPTIONS | ?PATCH | ?DELETE. -type url() :: binary(). -type error_code() :: ok | @@ -246,11 +245,89 @@ -type header() :: {binary(), iodata()}. -type headers() :: [header()]. -opaque cookiejar() :: [binary()]. --type doh_url() :: binary(). -type qs_vals() :: [{unicode:chardata(), unicode:chardata() | true}]. -type req_body() :: iodata() | qs_vals(). -type body() :: binary(). --type request() :: map(). +-type connecttimeout_ms() :: pos_integer(). +%% See [https://curl.haxx.se/libcurl/c/CURLOPT_CONNECTTIMEOUT.html] +-type ssl_verifyhost() :: boolean(). +%% See [https://curl.haxx.se/libcurl/c/CURLOPT_SSL_VERIFYHOST.html] +-type ssl_verifypeer() :: boolean(). +%% See [https://curl.haxx.se/libcurl/c/CURLOPT_SSL_VERIFYPEER.html] +-type proxy() :: binary(). +%% See [https://curl.haxx.se/libcurl/c/CURLOPT_PROXY.html] +-type tcp_fastopen() :: boolean(). +%% See [https://curl.haxx.se/libcurl/c/CURLOPT_TCP_FASTOPEN.html] +-type interface() :: binary(). +%% See [https://curl.haxx.se/libcurl/c/CURLOPT_INTERFACE.html] +-type unix_socket_path() :: binary(). +%% See [https://curl.haxx.se/libcurl/c/CURLOPT_UNIX_SOCKET_PATH.html] +-type doh_url() :: binary(). +%% See [https://curl.haxx.se/libcurl/c/CURLOPT_DOH_URL.html] +-type sslcert() :: binary(). +%% See [https://curl.haxx.se/libcurl/c/CURLOPT_SSLCERT.html] +-type sslkey() :: binary(). +%% See [https://curl.haxx.se/libcurl/c/CURLOPT_SSLKEY.html] +-type sslkey_blob() :: binary(). +%% See [https://curl.haxx.se/libcurl/c/CURLOPT_SSLKEY_BLOB.html] +-type userpwd() :: binary(). +%% See [https://curl.haxx.se/libcurl/c/CURLOPT_USERPWD.html] +-type request() :: #{url := binary(), + method := method(), + headers => headers(), + cookiejar => cookiejar(), + body => req_body(), + connecttimeout_ms => connecttimeout_ms(), + followlocation => boolean(), + ssl_verifyhost => ssl_verifyhost(), + ssl_verifypeer => ssl_verifypeer(), + capath => binary(), + cacert => binary(), + timeout_ms => pos_integer(), + maxredirs => non_neg_integer(), + http_auth => http_auth(), + username => binary(), + password => binary(), + proxy => proxy(), + return_metrics => boolean(), + tcp_fastopen => tcp_fastopen(), + interface => interface(), + unix_socket_path => unix_socket_path(), + lock_data_ssl_session => boolean(), + doh_url => doh_url(), + http_version => curlopt_http_version(), + verbose => boolean(), + sslcert => sslcert(), + sslkey => sslkey(), + sslkey_blob => sslkey_blob(), + userpwd => userpwd()}. +-type opts() :: #{headers => headers(), + cookiejar => cookiejar(), + body => req_body(), + connecttimeout_ms => connecttimeout_ms(), + followlocation => boolean(), + ssl_verifyhost => ssl_verifyhost(), + ssl_verifypeer => ssl_verifypeer(), + capath => binary(), + cacert => binary(), + timeout_ms => pos_integer(), + maxredirs => non_neg_integer(), + http_auth => http_auth(), + username => binary(), + password => binary(), + proxy => proxy(), + return_metrics => boolean(), + tcp_fastopen => tcp_fastopen(), + interface => interface(), + unix_socket_path => unix_socket_path(), + lock_data_ssl_session => boolean(), + doh_url => doh_url(), + http_version => curlopt_http_version(), + verbose => boolean(), + sslcert => sslcert(), + sslkey => sslkey(), + sslkey_blob => sslkey_blob(), + userpwd => userpwd()}. -type metrics() :: proplists:proplist(). -type response() :: {ok, #{status := status(), headers := headers(), @@ -260,7 +337,11 @@ {error, #{code := error_code(), message := error_msg()}}. -type http_auth() :: basic | digest | ntlm | negotiate. --type http_auth_int() :: ?CURLAUTH_UNDEFINED | ?CURLAUTH_BASIC | ?CURLAUTH_DIGEST | ?CURLAUTH_NTLM | ?CURLAUTH_NEGOTIATE. +-type http_auth_int() :: ?CURLAUTH_UNDEFINED | + ?CURLAUTH_BASIC | + ?CURLAUTH_DIGEST | + ?CURLAUTH_NTLM | + ?CURLAUTH_NEGOTIATE. -type pipelining() :: nothing | http1 | multiplex. -type curlopt_http_version() :: curl_http_version_none | curl_http_version_1_0 | @@ -268,6 +349,8 @@ curl_http_version_2_0 | curl_http_version_2tls | curl_http_version_2_prior_knowledge. +%% HTTP protocol version to use +%% see [https://curl.se/libcurl/c/CURLOPT_HTTP_VERSION.html] -type curlmopts() :: [{max_pipeline_length, non_neg_integer()} | {pipelining, pipelining()} | {max_total_connections, non_neg_integer()}]. @@ -287,9 +370,21 @@ -export_type([response/0]). -export_type([http_auth/0]). -export_type([curlmopts/0]). +-export_type([connecttimeout_ms/0]). +-export_type([ssl_verifyhost/0]). +-export_type([ssl_verifypeer/0]). +-export_type([proxy/0]). +-export_type([tcp_fastopen/0]). +-export_type([interface/0]). +-export_type([unix_socket_path/0]). +-export_type([doh_url/0]). +-export_type([sslcert/0]). +-export_type([sslkey/0]). +-export_type([sslkey_blob/0]). +-export_type([userpwd/0]). -record(req, { - method = ?get :: method_int(), + method = ?GET :: method_int(), url :: undefined | binary(), headers = [] :: headers(), cookiejar = [] :: cookiejar(), @@ -323,76 +418,90 @@ userpwd = undefined :: undefined | binary() }). +-type req() :: #req{}. + +%% @private tcp_fastopen_available() -> ?TCP_FASTOPEN_AVAILABLE. +%% @private unix_socket_path_available() -> ?UNIX_SOCKET_PATH_AVAILABLE. +%% @private doh_url_available() -> ?DOH_URL_AVAILABLE. +%% @private sslkey_blob_available() -> ?SSLKEY_BLOB_AVAILABLE. -dialyzer({nowarn_function, opt/3}). +%% @equiv get(Poolname, Url, #{}) -spec get(katipo_pool:name(), url()) -> response(). get(PoolName, Url) -> - get(PoolName, Url, #{}). + req(PoolName, #{url => Url, method => get}). --spec get(katipo_pool:name(), url(), request()) -> response(). +-spec get(katipo_pool:name(), url(), opts()) -> response(). get(PoolName, Url, Opts) -> req(PoolName, Opts#{url => Url, method => get}). +%% @equiv post(Poolname, Url, #{}) -spec post(katipo_pool:name(), url()) -> response(). post(PoolName, Url) -> - post(PoolName, Url, #{}). + req(PoolName, #{url => Url, method => post}). --spec post(katipo_pool:name(), url(), request()) -> response(). +-spec post(katipo_pool:name(), url(), opts()) -> response(). post(PoolName, Url, Opts) -> req(PoolName, Opts#{url => Url, method => post}). +%% @equiv put(Poolname, Url, #{}) -spec put(katipo_pool:name(), url()) -> response(). put(PoolName, Url) -> - put(PoolName, Url, #{}). + req(PoolName, #{url => Url, method => put}). --spec put(katipo_pool:name(), url(), request()) -> response(). +-spec put(katipo_pool:name(), url(), opts()) -> response(). put(PoolName, Url, Opts) -> req(PoolName, Opts#{url => Url, method => put}). +%% @equiv head(Poolname, Url, #{}) -spec head(katipo_pool:name(), url()) -> response(). head(PoolName, Url) -> - head(PoolName, Url, #{}). + req(PoolName, #{url => Url, method => head}). --spec head(katipo_pool:name(), url(), request()) -> response(). +-spec head(katipo_pool:name(), url(), opts()) -> response(). head(PoolName, Url, Opts) -> req(PoolName, Opts#{url => Url, method => head}). +%% @equiv options(Poolname, Url, #{}) -spec options(katipo_pool:name(), url()) -> response(). options(PoolName, Url) -> - options(PoolName, Url, #{}). + req(PoolName, #{url => Url, method => options}). --spec options(katipo_pool:name(), url(), request()) -> response(). +-spec options(katipo_pool:name(), url(), opts()) -> response(). options(PoolName, Url, Opts) -> req(PoolName, Opts#{url => Url, method => options}). +%% @equiv patch(Poolname, Url, #{}) -spec patch(katipo_pool:name(), url()) -> response(). patch(PoolName, Url) -> - patch(PoolName, Url, #{}). + req(PoolName, #{url => Url, method => patch}). --spec patch(katipo_pool:name(), url(), request()) -> response(). +-spec patch(katipo_pool:name(), url(), opts()) -> response(). patch(PoolName, Url, Opts) -> req(PoolName, Opts#{url => Url, method => patch}). +%% @equiv delete(Poolname, Url, #{}) -spec delete(katipo_pool:name(), url()) -> response(). delete(PoolName, Url) -> - delete(PoolName, Url, #{}). + req(PoolName, #{url => Url, method => delete}). --spec delete(katipo_pool:name(), url(), request()) -> response(). +-spec delete(katipo_pool:name(), url(), opts()) -> response(). delete(PoolName, Url, Opts) -> req(PoolName, Opts#{url => Url, method => delete}). +%% @private -spec req(katipo_pool:name(), request()) -> response(). req(PoolName, Opts) when is_map(Opts) -> @@ -412,14 +521,14 @@ req(PoolName, Opts) {error, _} = Error -> ok = katipo_metrics:notify_error(), Error - end; -req(_PoolName, Opts) -> - {error, #{code => bad_opts, message => Opts}}. + end. +%% @private start_link(CurlOpts) when is_list(CurlOpts) -> Args = [CurlOpts], gen_server:start_link(?MODULE, Args, []). +%% @private init([CurlOpts]) -> process_flag(trap_exit, true), case get_mopts(CurlOpts) of @@ -431,6 +540,7 @@ init([CurlOpts]) -> {stop, Error} end. +%% @private handle_call(#req{method = Method, url = Url, headers = Headers, @@ -464,40 +574,42 @@ handle_call(#req{method = Method, From, State=#state{port=Port, reqs=Reqs}) -> {Self, Ref} = From, - Opts = [{?connecttimeout_ms, ConnTimeoutMs}, - {?followlocation, FollowLocation}, - {?ssl_verifyhost, SslVerifyHost}, - {?ssl_verifypeer, SslVerifyPeer}, - {?capath, CAPath}, - {?cacert, CACert}, - {?timeout_ms, TimeoutMs}, - {?maxredirs, MaxRedirs}, - {?http_auth, HTTPAuth}, - {?username, Username}, - {?password, Password}, - {?proxy, Proxy}, - {?tcp_fastopen, TCPFastOpen}, - {?interface, Interface}, - {?unix_socket_path, UnixSocketPath}, - {?lock_data_ssl_session, LockDataSslSession}, - {?doh_url, DOHURL}, - {?http_version, HTTPVersion}, - {?verbose, Verbose}, - {?sslcert, SSLCert}, - {?sslkey, SSLKey}, - {?sslkey_blob, SSLKeyBlob}, - {?keypasswd, KeyPasswd}, - {?userpwd, UserPwd}], + Opts = [{?CONNECTTIMEOUT_MS, ConnTimeoutMs}, + {?FOLLOWLOCATION, FollowLocation}, + {?SSL_VERIFYHOST, SslVerifyHost}, + {?SSL_VERIFYPEER, SslVerifyPeer}, + {?CAPATH, CAPath}, + {?CACERT, CACert}, + {?TIMEOUT_MS, TimeoutMs}, + {?MAXREDIRS, MaxRedirs}, + {?HTTP_AUTH, HTTPAuth}, + {?USERNAME, Username}, + {?PASSWORD, Password}, + {?PROXY, Proxy}, + {?TCP_FASTOPEN, TCPFastOpen}, + {?INTERFACE, Interface}, + {?UNIX_SOCKET_PATH, UnixSocketPath}, + {?LOCK_DATA_SSL_SESSION, LockDataSslSession}, + {?DOH_URL, DOHURL}, + {?HTTP_VERSION, HTTPVersion}, + {?VERBOSE, Verbose}, + {?SSLCERT, SSLCert}, + {?SSLKEY, SSLKey}, + {?SSLKEY_BLOB, SSLKeyBlob}, + {?KEYPASSWD, KeyPasswd}, + {?USERPWD, UserPwd}], Command = {Self, Ref, Method, Url, Headers, CookieJar, Body, Opts}, true = port_command(Port, term_to_binary(Command)), Tref = erlang:start_timer(Timeout, self(), {req_timeout, From}), Reqs2 = maps:put(From, Tref, Reqs), {noreply, State#state{reqs=Reqs2}}. +%% @private handle_cast(Msg, State) -> error_logger:error_msg("Unexpected cast: ~p", [Msg]), {noreply, State}. +%% @private handle_info({Port, {data, Data}}, State=#state{port=Port, reqs=Reqs}) -> {Result, {From, Response}} = case binary_to_term(Data) of @@ -536,10 +648,12 @@ handle_info({'EXIT', Port, Reason}, State=#state{port=Port}) -> error_logger:error_msg("Port ~p died with reason: ~p", [Port, Reason]), {stop, port_died, State}. +%% @private terminate(_Reason, #state{port=Port}) -> true = port_close(Port), ok. +%% @private code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -548,13 +662,13 @@ headers_to_binary(Headers) -> [iolist_to_binary([K, <<": ">>, V]) || {K, V} <- Headers]. -spec method_to_int(method()) -> method_int(). -method_to_int(get) -> ?get; -method_to_int(post) -> ?post; -method_to_int(put) -> ?put; -method_to_int(head) -> ?head; -method_to_int(options) -> ?options; -method_to_int(patch) -> ?patch; -method_to_int(delete) -> ?delete. +method_to_int(get) -> ?GET; +method_to_int(post) -> ?POST; +method_to_int(put) -> ?PUT; +method_to_int(head) -> ?HEAD; +method_to_int(options) -> ?OPTIONS; +method_to_int(patch) -> ?PATCH; +method_to_int(delete) -> ?DELETE. -spec parse_headers([binary()]) -> headers(). parse_headers([_StatusLine | Lines]) -> @@ -607,7 +721,8 @@ mopt_supported({max_total_connections, Val}) mopt_supported({_, _}) -> false. --spec get_timeout(#req{}) -> pos_integer(). +%% @private +-spec get_timeout(req()) -> pos_integer(). get_timeout(#req{connecttimeout_ms=ConnMs, timeout_ms=ReqMs}) -> max(ConnMs, ReqMs). @@ -719,7 +834,7 @@ opt(userpwd, UserPwd, {Req, Errors}) when is_binary(UserPwd) -> opt(K, V, {Req, Errors}) -> {Req, [{K, V} | Errors]}. --spec process_opts(request()) -> {ok, #req{}} | {error, map()}. +-spec process_opts(request()) -> {ok, req()} | {error, map()}. process_opts(Opts) -> case maps:fold(fun opt/3, {#req{}, []}, Opts) of {Req=#req{}, []} -> @@ -728,6 +843,7 @@ process_opts(Opts) -> {error, error_map(bad_opts, Errors)} end. +%% @private -spec check_opts(request()) -> ok | {error, map()}. check_opts(Opts) when is_map(Opts) -> case process_opts(Opts) of diff --git a/src/katipo_metrics.erl b/src/katipo_metrics.erl index b5aa457..362603d 100644 --- a/src/katipo_metrics.erl +++ b/src/katipo_metrics.erl @@ -1,3 +1,4 @@ +%% @hidden -module(katipo_metrics). -export([init/0]). diff --git a/src/katipo_session.erl b/src/katipo_session.erl deleted file mode 100644 index a27b145..0000000 --- a/src/katipo_session.erl +++ /dev/null @@ -1,71 +0,0 @@ --module(katipo_session). - --export([new/1]). --export([new/2]). --export([update/2]). --export([req/2]). - --record(state, {pool_name :: katipo_pool:name(), - opts = #{} :: katipo:request()}). - --opaque session() :: #state{}. - --type session_result() :: {ok, session()} | {error, map()}. - --export_type([session/0]). --export_type([session_result/0]). - --spec new(katipo_pool:name()) -> session_result(). -new(PoolName) -> - new(PoolName, #{}). - --spec new(katipo_pool:name(), katipo:request()) -> session_result(). -new(PoolName, Opts) when is_atom(PoolName) andalso is_map(Opts) -> - State = #state{pool_name=PoolName, opts = Opts}, - check_opts(State). - --spec update(katipo:request(), session()) -> session_result(). -update(Opts, State=#state{}) -> - Opts2 = merge(State#state.opts, Opts), - State2 = State#state{opts=Opts2}, - check_opts(State2). - -check_opts(State=#state{opts=Opts}) -> - case katipo:check_opts(Opts) of - ok -> - {ok, State}; - {error, _} = Error -> - Error - end. - --spec req(katipo:request(), session()) -> {katipo:response(), session()}. -req(Req, State=#state{pool_name=PoolName, opts=Opts}) when is_map(Req) -> - Req2 = merge(Opts, Req), - Res = katipo:req(PoolName, Req2), - Opts2 = case Res of - {ok, #{cookiejar := CookieJar}} -> - Opts#{cookiejar => CookieJar}; - {error, #{}} -> - Opts - end, - {Res, State#state{opts=Opts2}}. - -merge(Opts, Req) when is_map(Req) andalso is_map(Opts) -> - Merged = maps:merge(Opts, Req), - case maps:get(headers, Req, undefined) of - undefined -> - Merged; - ReqHeaders -> - OptsHeaders = maps:get(headers, Opts, []), - MergedHeaders = merge_headers(OptsHeaders, ReqHeaders), - Merged#{headers => MergedHeaders} - end. - -merge_headers(OptsHeaders, ReqHeaders) -> - merge_headers(OptsHeaders, ReqHeaders, OptsHeaders). - -merge_headers(_, [], Merged) -> - Merged; -merge_headers(OptsHeaders, [{K, _V}=H|Rest], Merged) -> - Merged2 = lists:keyreplace(K, 1, Merged, H), - merge_headers(OptsHeaders, Rest, Merged2). diff --git a/src/katipo_sup.erl b/src/katipo_sup.erl index 81b2ba0..a46fc99 100644 --- a/src/katipo_sup.erl +++ b/src/katipo_sup.erl @@ -1,3 +1,4 @@ +%% @hidden -module(katipo_sup). -behaviour(supervisor). diff --git a/test/katipo_SUITE.erl b/test/katipo_SUITE.erl index 6c1b98a..bdddf33 100644 --- a/test/katipo_SUITE.erl +++ b/test/katipo_SUITE.erl @@ -26,9 +26,6 @@ init_per_group(http, Config) -> {ok, _} = cowboy:start_clear(unix_socket, [{ip, {local, Filename}}, {port, 0}], #{env => #{dispatch => Dispatch}}), [{unix_socket_file, Filename} | Config]; -init_per_group(session, Config) -> - application:ensure_all_started(katipo), - Config; init_per_group(pool, Config) -> application:ensure_all_started(meck), Config; @@ -151,13 +148,6 @@ groups() -> badssl]}, {https_mutual, [], [badssl_client_cert]}, - {session, [parallel], - [session_new, - session_new_bad_opts, - session_new_cookies, - session_new_headers, - session_update, - session_update_bad_opts]}, {port, [], [max_total_connections]}, {metrics, [], @@ -172,7 +162,6 @@ all() -> {group, pool}, {group, https}, {group, https_mutual}, - {group, session}, {group, port}, {group, metrics}, {group, http2}]. @@ -180,19 +169,19 @@ all() -> get(_) -> {ok, #{status := 200, body := Body}} = katipo:get(?POOL, <<"https://httpbin.org/get?a=%21%40%23%24%25%5E%26%2A%28%29_%2B">>), - Json = jsx:decode(Body), + Json = jsx:decode(Body, [{return_maps, false}]), [{<<"a">>, <<"!@#$%^&*()_+">>}] = proplists:get_value(<<"args">>, Json). get_http(_) -> {ok, #{status := 200, body := Body}} = katipo:get(?POOL, <<"http://httpbin.org/get?a=%21%40%23%24%25%5E%26%2A%28%29_%2B">>), - Json = jsx:decode(Body), + Json = jsx:decode(Body, [{return_maps, false}]), [{<<"a">>, <<"!@#$%^&*()_+">>}] = proplists:get_value(<<"args">>, Json). get_req(_) -> {ok, #{status := 200, body := Body}} = katipo:req(?POOL, #{url => <<"https://httpbin.org/get?a=%21%40%23%24%25%5E%26%2A%28%29_%2B">>}), - Json = jsx:decode(Body), + Json = jsx:decode(Body, [{return_maps, false}]), [{<<"a">>, <<"!@#$%^&*()_+">>}] = proplists:get_value(<<"args">>, Json). head(_) -> @@ -204,7 +193,7 @@ post_body_binary(_) -> katipo:post(?POOL, <<"https://httpbin.org/post">>, #{headers => [{<<"Content-Type">>, <<"application/json">>}], body => <<"!@#$%^&*()">>}), - Json = jsx:decode(Body), + Json = jsx:decode(Body, [{return_maps, false}]), <<"!@#$%^&*()">> = proplists:get_value(<<"data">>, Json). post_body_iolist(_) -> @@ -212,7 +201,7 @@ post_body_iolist(_) -> katipo:post(?POOL, <<"https://httpbin.org/post">>, #{headers => [{<<"Content-Type">>, <<"application/json">>}], body => ["foo", $b, $a, $r, <<"baz">>]}), - Json = jsx:decode(Body), + Json = jsx:decode(Body, [{return_maps, false}]), <<"foobarbaz">> = proplists:get_value(<<"data">>, Json). post_body_qs_vals(_) -> @@ -220,7 +209,7 @@ post_body_qs_vals(_) -> katipo:post(?POOL, <<"https://httpbin.org/post">>, #{headers => [{<<"Content-Type">>, <<"application/json">>}], body => [<<"!@#$%">>, <<"^&*()">>]}), - Json = jsx:decode(Body), + Json = jsx:decode(Body, [{return_maps, false}]), <<"!@#$%^&*()">> = proplists:get_value(<<"data">>, Json). post_body_bad(_) -> @@ -234,14 +223,14 @@ post_body_bad(_) -> post_arity_2(_) -> {ok, #{status := 200, body := Body}} = katipo:post(?POOL, <<"https://httpbin.org/post">>), - Json = jsx:decode(Body), + Json = jsx:decode(Body, [{return_maps, false}]), undefined = proplists:get_value(<<>>, Json). post_qs(_) -> QsVals = [{<<"foo">>, <<"bar">>}, {<<"baz">>, true}], {ok, #{status := 200, body := Body}} = katipo:post(?POOL, <<"https://httpbin.org/post">>, #{body => QsVals}), - Json = jsx:decode(Body), + Json = jsx:decode(Body, [{return_maps, false}]), [] = [{<<"baz">>,<<>>},{<<"foo">>,<<"bar">>}] -- proplists:get_value(<<"form">>, Json). post_qs_invalid(_) -> @@ -255,7 +244,7 @@ post_req(_) -> method => post, headers => [{<<"Content-Type">>, <<"application/json">>}], body => <<"!@#$%^&*()">>}), - Json = jsx:decode(Body), + Json = jsx:decode(Body, [{return_maps, false}]), <<"!@#$%^&*()">> = proplists:get_value(<<"data">>, Json). url_missing(_) -> @@ -279,20 +268,20 @@ put_data(_) -> {ok, #{status := 200, body := Body}} = katipo:put(?POOL, <<"https://httpbin.org/put">>, #{headers => Headers, body => <<"!@#$%^&*()">>}), - Json = jsx:decode(Body), + Json = jsx:decode(Body, [{return_maps, false}]), <<"!@#$%^&*()">> = proplists:get_value(<<"data">>, Json). put_arity_2(_) -> {ok, #{status := 200, body := Body}} = katipo:put(?POOL, <<"https://httpbin.org/put">>), - Json = jsx:decode(Body), + Json = jsx:decode(Body, [{return_maps, false}]), undefined = proplists:get_value(<<>>, Json). put_qs(_) -> QsVals = [{<<"foo">>, <<"bar">>}, {<<"baz">>, true}], {ok, #{status := 200, body := Body}} = katipo:put(?POOL, <<"https://httpbin.org/put">>, #{body => QsVals}), - Json = jsx:decode(Body), + Json = jsx:decode(Body, [{return_maps, false}]), [] = [{<<"baz">>,<<>>},{<<"foo">>,<<"bar">>}] -- proplists:get_value(<<"form">>, Json). patch_data(_) -> @@ -300,20 +289,20 @@ patch_data(_) -> {ok, #{status := 200, body := Body}} = katipo:patch(?POOL, <<"https://httpbin.org/patch">>, #{headers => Headers, body => <<"!@#$%^&*()">>}), - Json = jsx:decode(Body), + Json = jsx:decode(Body, [{return_maps, false}]), <<"!@#$%^&*()">> = proplists:get_value(<<"data">>, Json). patch_arity_2(_) -> {ok, #{status := 200, body := Body}} = katipo:patch(?POOL, <<"https://httpbin.org/patch">>), - Json = jsx:decode(Body), + Json = jsx:decode(Body, [{return_maps, false}]), <<>> = proplists:get_value(<<"data">>, Json). patch_qs(_) -> QsVals = [{<<"foo">>, <<"bar">>}, {<<"baz">>, true}], {ok, #{status := 200, body := Body}} = katipo:patch(?POOL, <<"https://httpbin.org/patch">>, #{body => QsVals}), - Json = jsx:decode(Body), + Json = jsx:decode(Body, [{return_maps, false}]), [] = [{<<"baz">>,<<>>},{<<"foo">>,<<"bar">>}] -- proplists:get_value(<<"form">>, Json). options(_) -> @@ -329,7 +318,7 @@ headers(_) -> Headers = [{<<"header1">>, <<"!@#$%^&*()">>}], {ok, #{status := 200, body := Body}} = katipo:get(?POOL, <<"https://httpbin.org/gzip">>, #{headers => Headers}), - Json = jsx:decode(Body), + Json = jsx:decode(Body, [{return_maps, false}]), Expected = [{<<"Accept">>,<<"*/*">>}, {<<"Accept-Encoding">>,<<"gzip,deflate">>}, {<<"Header1">>,<<"!@#$%^&*()">>}, @@ -340,19 +329,19 @@ header_remove(_) -> Headers = [{<<"Accept-Encoding">>, <<>>}], {ok, #{status := 200, body := Body}} = katipo:get(?POOL, <<"https://httpbin.org/get">>, #{headers => Headers}), - Json = jsx:decode(Body), + Json = jsx:decode(Body, [{return_maps, false}]), Expected = [{<<"Accept">>,<<"*/*">>}, {<<"Host">>,<<"httpbin.org">>}], [] = Expected -- proplists:get_value(<<"headers">>, Json). gzip(_) -> {ok, #{status := 200, body := Body}} = katipo:get(?POOL, <<"https://httpbin.org/gzip">>), - Json = jsx:decode(Body), + Json = jsx:decode(Body, [{return_maps, false}]), true = proplists:get_value(<<"gzipped">>, Json). deflate(_) -> {ok, #{status := 200, body := Body}} = katipo:get(?POOL, <<"https://httpbin.org/deflate">>), - Json = jsx:decode(Body), + Json = jsx:decode(Body, [{return_maps, false}]), true = proplists:get_value(<<"deflated">>, Json). bytes(_) -> @@ -389,7 +378,7 @@ cookies(_) -> Url = <<"https://httpbin.org/cookies/set?cname=cvalue">>, Opts = #{followlocation => true}, {ok, #{status := 200, body := Body}} = katipo:get(?POOL, Url, Opts), - Json = jsx:decode(Body), + Json = jsx:decode(Body, [{return_maps, false}]), [{<<"cname">>, <<"cvalue">>}] = proplists:get_value(<<"cookies">>, Json). cookies_delete(_) -> @@ -399,7 +388,7 @@ cookies_delete(_) -> DeleteUrl = <<"https://httpbin.org/cookies/delete?cname">>, {ok, #{status := 200, body := Body}} = katipo:get(?POOL, DeleteUrl, #{cookiejar => CookieJar, followlocation => true}), - Json = jsx:decode(Body), + Json = jsx:decode(Body, [{return_maps, false}]), [{}] = proplists:get_value(<<"cookies">>, Json). cookies_bad_cookie_jar(_) -> @@ -497,7 +486,7 @@ basic_authorised(_) -> {ok, #{status := 200, body := Body}} = katipo:get(?POOL, <<"https://httpbin.org/basic-auth/johndoe/p455w0rd">>, #{http_auth => basic, username => Username, password => Password}), - Json = jsx:decode(Body), + Json = jsx:decode(Body, [{return_maps, false}]), true = proplists:get_value(<<"authenticated">>, Json), Username = proplists:get_value(<<"user">>, Json). @@ -507,7 +496,7 @@ basic_authorised_userpwd(_) -> {ok, #{status := 200, body := Body}} = katipo:get(?POOL, <<"https://httpbin.org/basic-auth/johndoe/p455w0rd">>, #{http_auth => basic, userpwd => <>}), - Json = jsx:decode(Body), + Json = jsx:decode(Body, [{return_maps, false}]), true = proplists:get_value(<<"authenticated">>, Json), Username = proplists:get_value(<<"user">>, Json). @@ -521,7 +510,7 @@ digest_authorised(_) -> {ok, #{status := 200, body := Body}} = katipo:get(?POOL, <<"https://httpbin.org/digest-auth/auth/johndoe/p455w0rd">>, #{http_auth => digest, username => Username, password => Password}), - Json = jsx:decode(Body), + Json = jsx:decode(Body, [{return_maps, false}]), true = proplists:get_value(<<"authenticated">>, Json), Username = proplists:get_value(<<"user">>, Json). @@ -531,7 +520,7 @@ digest_authorised_userpwd(_) -> {ok, #{status := 200, body := Body}} = katipo:get(?POOL, <<"https://httpbin.org/digest-auth/auth/johndoe/p455w0rd">>, #{http_auth => digest, userpwd => <>}), - Json = jsx:decode(Body), + Json = jsx:decode(Body, [{return_maps, false}]), true = proplists:get_value(<<"authenticated">>, Json), Username = proplists:get_value(<<"user">>, Json). @@ -539,14 +528,14 @@ lock_data_ssl_session_true(_) -> {ok, #{status := 200, body := Body}} = katipo:get(?POOL, <<"https://httpbin.org/get?a=%21%40%23%24%25%5E%26%2A%28%29_%2B">>, #{lock_data_ssl_session => true}), - Json = jsx:decode(Body), + Json = jsx:decode(Body, [{return_maps, false}]), [{<<"a">>, <<"!@#$%^&*()_+">>}] = proplists:get_value(<<"args">>, Json). lock_data_ssl_session_false(_) -> {ok, #{status := 200, body := Body}} = katipo:get(?POOL, <<"https://httpbin.org/get?a=%21%40%23%24%25%5E%26%2A%28%29_%2B">>, #{lock_data_ssl_session => false}), - Json = jsx:decode(Body), + Json = jsx:decode(Body, [{return_maps, false}]), [{<<"a">>, <<"!@#$%^&*()_+">>}] = proplists:get_value(<<"args">>, Json). doh_url(_) -> @@ -762,74 +751,6 @@ badssl_client_cert(Config) -> end, ok. -%% session - -session_new(_) -> - {ok, Session} = katipo_session:new(?POOL), - Url = <<"https://httpbin.org/cookies/set?cname=cvalue">>, - Req = #{url => Url, followlocation => true}, - {{ok, #{status := 200, cookiejar := CookieJar, body := Body}}, Session2} = - katipo_session:req(Req, Session), - {state, ?POOL, #{cookiejar := CookieJar}} = Session2, - Json = jsx:decode(Body), - [{<<"cname">>, <<"cvalue">>}] = proplists:get_value(<<"cookies">>, Json). - -session_new_bad_opts(_) -> - {error, #{code := bad_opts}} = - katipo_session:new(?POOL, #{timeout_ms => <<"wrong">>, what => not_even_close}). - -session_new_cookies(_) -> - Url = <<"https://httpbin.org/cookies/delete?cname">>, - CookieJar = [<<"httpbin.org\tFALSE\t/\tTRUE\t0\tcname\tcvalue">>, - <<"httpbin.org\tFALSE\t/\tTRUE\t0\tcname2\tcvalue2">>], - Req = #{url => Url, cookiejar => CookieJar, followlocation => true}, - {ok, Session} = katipo_session:new(?POOL, Req), - {{ok, #{status := 200, body := Body}}, Session2} = - katipo_session:req(#{}, Session), - Json = jsx:decode(Body), - [{<<"cname2">>, <<"cvalue2">>}] = proplists:get_value(<<"cookies">>, Json), - Url2 = <<"https://httpbin.org/cookies/delete?cname2">>, - {{ok, #{status := 200, body := Body2}}, _} = - katipo_session:req(#{url => Url2}, Session2), - Json2 = jsx:decode(Body2), - [{}] = proplists:get_value(<<"cookies">>, Json2). - -session_new_headers(_) -> - Req = #{url => <<"https://httpbin.org/cookies/delete?cname">>, - headers => [{<<"header1">>, <<"dontcare">>}]}, - {ok, Session} = katipo_session:new(?POOL, Req), - {{ok, #{status := 200, body := Body}}, _Session2} = - katipo_session:req(#{url => <<"https://httpbin.org/gzip">>, - headers => [{<<"header1">>, <<"!@#$%^&*()">>}]}, - Session), - Json = jsx:decode(Body), - Expected = [{<<"Accept">>,<<"*/*">>}, - {<<"Accept-Encoding">>,<<"gzip,deflate">>}, - {<<"Header1">>,<<"!@#$%^&*()">>}, - {<<"Host">>,<<"httpbin.org">>}], - [] = Expected -- proplists:get_value(<<"headers">>, Json). - -session_update(_) -> - Req = #{url => <<"https://httpbin.org/cookies/delete?cname">>, - headers => [{<<"header1">>, <<"dontcare">>}]}, - {ok, Session} = katipo_session:new(?POOL, Req), - Req2 = #{url => <<"https://httpbin.org/gzip">>, - headers => [{<<"header1">>, <<"!@#$%^&*()">>}]}, - {ok, Session2} = katipo_session:update(Req2, Session), - {{ok, #{status := 200, body := Body}}, _Session3} = - katipo_session:req(#{}, Session2), - Json = jsx:decode(Body), - Expected = [{<<"Accept">>,<<"*/*">>}, - {<<"Accept-Encoding">>,<<"gzip,deflate">>}, - {<<"Header1">>,<<"!@#$%^&*()">>}, - {<<"Host">>,<<"httpbin.org">>}], - [] = Expected -- proplists:get_value(<<"headers">>, Json). - -session_update_bad_opts(_) -> - {ok, Session} = katipo_session:new(?POOL), - {error, #{code := bad_opts}} = - katipo_session:update(#{timeout_ms => <<"wrong">>, what => not_even_close}, Session). - max_total_connections(_) -> PoolName = max_total_connections, {ok, _} = katipo_pool:start(PoolName, 1, [{pipelining, nothing}, {max_total_connections, 1}]), @@ -881,7 +802,7 @@ http2_get(_) -> {ok, #{status := 200, body := Body}} = katipo:get(?POOL, <<"https://nghttp2.org/httpbin/get?a=%21%40%23%24%25%5E%26%2A%28%29_%2B">>, #{http_version => curl_http_version_2_prior_knowledge}), - Json = jsx:decode(Body), + Json = jsx:decode(Body, [{return_maps, false}]), [{<<"a">>, <<"!@#$%^&*()_+">>}] = proplists:get_value(<<"args">>, Json). repeat_until_true(Fun) ->