From 76176487ba3b88d03231e0bb7225b825db50f72f Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Thu, 18 Apr 2024 17:33:51 +0900 Subject: [PATCH] Add SSL options to configuration (#114) * Add development guide for schema registry * Bump version to 0.28.0 --- CHANGELOG.md | 47 +++--- DEVELOPMENT.md | 208 ++++++++++++++++++++++++++ README.md | 19 ++- lib/avrora/client.ex | 7 + lib/avrora/config.ex | 16 ++ lib/avrora/http_client.ex | 10 +- lib/avrora/storage/registry.ex | 23 ++- mix.exs | 2 +- test/avrora/storage/registry_test.exs | 170 +++++++++++++-------- test/support/config.ex | 4 + 10 files changed, 407 insertions(+), 99 deletions(-) create mode 100644 DEVELOPMENT.md diff --git a/CHANGELOG.md b/CHANGELOG.md index d2610ca7..dcb4ba71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,45 +1,42 @@ # Changelog -## [0.27.0] - 2023-07-14 ([notes][0.27.0-n]) +## [0.28.0] - 2024-04-18 + +- Add new `Avrora.Config` SSL options `registry_ssl_cacerts` and `registry_ssl_cacert_path` (#114 @strech) + +## [0.27.0] - 2023-07-14 - Replace `Logger.warn/1` with `Logger.warning/2` (#107 @trbngr, @strech) - Drop support for Elixir lower than 1.12 (#107 @strech) -## [0.26.0] - 2023-01-11 ([notes][0.26.0-n]) +## [0.26.0] - 2023-01-11 - Add `--appconfig` argument to schema registration mix task (#102 @emilianobovetti, @strech) -## [0.25.0] - 2023-01-03 ([notes][0.25.0-n]) +## [0.25.0] - 2023-01-03 -- Add User-Agent header when communicating with Schema Registry (#100 @azeemchauhan, @strech) +- Add `User-Agent` header when communicating with Schema Registry (#100 @azeemchauhan, @strech) -## [0.24.2] - 2022-09-13 ([notes][0.24.2-n]) +## [0.24.2] - 2022-09-13 -- Fix Avrora.Config.registry_schemas_autoreg/0 to return configured `false` value (#99 @ankhers) +- Fix `Avrora.Config.registry_schemas_autoreg/0` to return configured `false` value (#99 @ankhers) -## [0.24.1] - 2022-09-12 ([notes][0.24.1-n]) +## [0.24.1] - 2022-09-12 -- Add SSL option `[verify: :verify_none]` to Avrora.HttpClient (#97, @goozzik) +- Add SSL option `[verify: :verify_none]` to `Avrora.HttpClient` (#97, @goozzik) -## [0.24.0] - 2022-03-16 ([notes][0.24.0-n]) +## [0.24.0] - 2022-03-16 -- Add new Avrora.Config option decoder_hook (#94, @strech) +- Add new `Avrora.Config` option decoder_hook (#94, @strech) -## [0.23.0] - 2021-07-06 ([notes][0.23.0-n]) +## [0.23.0] - 2021-07-06 - Add runtime config resolution for Avrora.Client (#92, @strech) -[0.27.0]: https://github.com/Strech/avrora/compare/v0.26.0...v0.27.0 -[0.27.0-n]: https://github.com/Strech/avrora/releases/tag/v0.27.0 -[0.26.0]: https://github.com/Strech/avrora/compare/v0.25.0...v0.26.0 -[0.26.0-n]: https://github.com/Strech/avrora/releases/tag/v0.26.0 -[0.25.0]: https://github.com/Strech/avrora/compare/v0.24.2...v0.25.0 -[0.25.0-n]: https://github.com/Strech/avrora/releases/tag/v0.25.0 -[0.24.2]: https://github.com/Strech/avrora/compare/v0.24.1...v0.24.2 -[0.24.2-n]: https://github.com/Strech/avrora/releases/tag/v0.24.2 -[0.24.1]: https://github.com/Strech/avrora/compare/v0.24.0...v0.24.1 -[0.24.1-n]: https://github.com/Strech/avrora/releases/tag/v0.24.1 -[0.24.0]: https://github.com/Strech/avrora/compare/v0.23.0...v0.24.0 -[0.24.0-n]: https://github.com/Strech/avrora/releases/tag/v0.24.0 -[0.23.0]: https://github.com/Strech/avrora/compare/v0.22.0...v0.23.0 -[0.23.0-n]: https://github.com/Strech/avrora/releases/tag/v0.23.0 +[0.27.0]: https://github.com/Strech/avrora/releases/tag/v0.27.0 +[0.26.0]: https://github.com/Strech/avrora/releases/tag/v0.26.0 +[0.25.0]: https://github.com/Strech/avrora/releases/tag/v0.25.0 +[0.24.2]: https://github.com/Strech/avrora/releases/tag/v0.24.2 +[0.24.1]: https://github.com/Strech/avrora/releases/tag/v0.24.1 +[0.24.0]: https://github.com/Strech/avrora/releases/tag/v0.24.0 +[0.23.0]: https://github.com/Strech/avrora/releases/tag/v0.23.0 diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 00000000..d4ea7bd0 --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,208 @@ +# Development + +To test `Avrora` with real Confluent Schema Registry it is recommended to use +official demo project. + +## 1. Setup Confluent demo project + +```console +$ git clone git clone git@github.com:confluentinc/cp-demo.git +$ cd cp-demo +$ git checkout -b 7.5.1-post origin/7.5.1-post +``` + +Apply the following patch to minimize the setup and keep the bare-minimum. + +> [!NOTE] +> You may see some errors and warnings, but you could ignore them, for example: +> +> Error response from daemon: No such container: connect +> WARNING: Expected to find at least 6 subjects in Schema Registry but found... + +```diff +diff --git a/scripts/start.sh b/scripts/start.sh +index 50b5b41a..535b7a0e 100755 +--- a/scripts/start.sh ++++ b/scripts/start.sh +@@ -5,6 +5,8 @@ source ${DIR}/helper/functions.sh + source ${DIR}/env.sh + + #------------------------------------------------------------------------------- ++# Disable visualization by default ++VIZ=false + + # Do preflight checks + preflight_checks || exit +@@ -15,7 +17,7 @@ ${DIR}/stop.sh + CLEAN=${CLEAN:-false} + + # Build Kafka Connect image with connector plugins +-build_connect_image ++# build_connect_image + + # Set the CLEAN variable to true if cert doesn't exist + if ! [[ -f "${DIR}/security/controlCenterAndKsqlDBServer-ca1-signed.crt" ]] || ! check_num_certs; then +@@ -86,75 +88,82 @@ docker-compose exec kafka1 kafka-configs \ + + + # Bring up more containers +-docker-compose up --no-recreate -d schemaregistry connect control-center ++# FIXME ++# docker-compose up --no-recreate -d schemaregistry connect control-center ++docker-compose up --no-recreate -d schemaregistry + + echo + echo -e "Create topics in Kafka cluster:" + docker-compose exec tools bash -c "/tmp/helper/create-topics.sh" || exit 1 + + # Verify Kafka Connect Worker has started +-MAX_WAIT=240 +-echo -e "\nWaiting up to $MAX_WAIT seconds for Connect to start" +-retry $MAX_WAIT host_check_up connect || exit 1 ++# FIXME ++# MAX_WAIT=240 ++# echo -e "\nWaiting up to $MAX_WAIT seconds for Connect to start" ++# retry $MAX_WAIT host_check_up connect || exit 1 + + #------------------------------------------------------------------------------- + +-echo -e "\nStart streaming from the Wikipedia SSE source connector:" +-${DIR}/connectors/submit_wikipedia_sse_config.sh || exit 1 ++# FIXME ++# echo -e "\nStart streaming from the Wikipedia SSE source connector:" ++# ${DIR}/connectors/submit_wikipedia_sse_config.sh || exit 1 + +-# Verify connector is running +-MAX_WAIT=120 +-echo +-echo "Waiting up to $MAX_WAIT seconds for connector to be in RUNNING state" +-retry $MAX_WAIT check_connector_status_running "wikipedia-sse" || exit 1 ++# # Verify connector is running ++# MAX_WAIT=120 ++# echo ++# echo "Waiting up to $MAX_WAIT seconds for connector to be in RUNNING state" ++# retry $MAX_WAIT check_connector_status_running "wikipedia-sse" || exit 1 + +-# Verify wikipedia.parsed topic is populated and schema is registered +-MAX_WAIT=120 +-echo +-echo -e "Waiting up to $MAX_WAIT seconds for subject wikipedia.parsed-value (for topic wikipedia.parsed) to be registered in Schema Registry" +-retry $MAX_WAIT host_check_schema_registered || exit 1 ++# # Verify wikipedia.parsed topic is populated and schema is registered ++# MAX_WAIT=120 ++# echo ++# echo -e "Waiting up to $MAX_WAIT seconds for subject wikipedia.parsed-value (for topic wikipedia.parsed) to be registered in Schema Registry" ++# retry $MAX_WAIT host_check_schema_registered || exit 1 + + #------------------------------------------------------------------------------- + +-# Verify Confluent Control Center has started +-MAX_WAIT=300 +-echo +-echo "Waiting up to $MAX_WAIT seconds for Confluent Control Center to start" +-retry $MAX_WAIT host_check_up control-center || exit 1 ++# # Verify Confluent Control Center has started ++# FIXME ++# MAX_WAIT=300 ++# echo ++# echo "Waiting up to $MAX_WAIT seconds for Confluent Control Center to start" ++# retry $MAX_WAIT host_check_up control-center || exit 1 + +-echo -e "\nConfluent Control Center modifications:" +-${DIR}/helper/control-center-modifications.sh +-echo ++# echo -e "\nConfluent Control Center modifications:" ++# ${DIR}/helper/control-center-modifications.sh ++# echo + + + #------------------------------------------------------------------------------- + +-# Start more containers +-docker-compose up --no-recreate -d ksqldb-server ksqldb-cli restproxy ++# FIXME ++# # Start more containers ++# docker-compose up --no-recreate -d ksqldb-server ksqldb-cli restproxy + +-# Verify ksqlDB server has started +-echo +-echo +-MAX_WAIT=120 +-echo -e "\nWaiting up to $MAX_WAIT seconds for ksqlDB server to start" +-retry $MAX_WAIT host_check_up ksqldb-server || exit 1 ++# # Verify ksqlDB server has started ++# echo ++# echo ++# MAX_WAIT=120 ++# echo -e "\nWaiting up to $MAX_WAIT seconds for ksqlDB server to start" ++# retry $MAX_WAIT host_check_up ksqldb-server || exit 1 + +-echo -e "\nRun ksqlDB queries:" +-${DIR}/ksqlDB/run_ksqlDB.sh ++# echo -e "\nRun ksqlDB queries:" ++# ${DIR}/ksqlDB/run_ksqlDB.sh + + if [[ "$VIZ" == "true" ]]; then + build_viz || exit 1 + fi + +-echo -e "\nStart additional consumers to read from topics WIKIPEDIANOBOT, WIKIPEDIA_COUNT_GT_1" +-${DIR}/consumers/listen_WIKIPEDIANOBOT.sh +-${DIR}/consumers/listen_WIKIPEDIA_COUNT_GT_1.sh ++# FIXME ++# echo -e "\nStart additional consumers to read from topics WIKIPEDIANOBOT, WIKIPEDIA_COUNT_GT_1" ++# ${DIR}/consumers/listen_WIKIPEDIANOBOT.sh ++# ${DIR}/consumers/listen_WIKIPEDIA_COUNT_GT_1.sh + +-echo +-echo +-echo "Start the Kafka Streams application wikipedia-activity-monitor" +-docker-compose up --no-recreate -d streams-demo +-echo "..." ++# echo ++# echo ++# echo "Start the Kafka Streams application wikipedia-activity-monitor" ++# docker-compose up --no-recreate -d streams-demo ++# echo "..." + + + #------------------------------------------------------------------------------- +``` + +Save it with name `cp-demo.patch` and run the following command. + +```console +$ git apply cp-demo.patch +``` + +To ensure HTTP(S) connectivity with the generated certificate, +add the schema registry to your system's hosts file: + +```console +$ sudo sh -c 'echo "0.0.0.0 schemaregistry" >> /etc/hosts' +``` + +## 2. Get certificates + +Copy the certificate from the Docker container to your local machine and +convert the PEM certificate to a DER-encoded format. + +```console +$ docker cp schemaregistry:/etc/kafka/secrets/snakeoil-ca-1.crt . +$ openssl x509 -in snakeoil-ca-1.crt -outform DER -out snakeoil-ca-1.der +``` + +## 3. Check Schema Registry connectivity + +Test the connection to the Schema Registry using the `Avrora.HTTPClient` +with the converted certificate. Run the following code in your console. + +```elixir +# Setup the URL and read the certificate +url = "https://superUser:superUser@schemaregistry:8085/subjects" +cert = File.read!(Path.expand("./snakeoil-ca-1.der")) + +# Make a get request to the Schema Registry it should output `{:ok, []}` +# (because no data was populated in patched `start.sh` script) +Avrora.HTTPClient.get(url, [ssl_options: [verify: :verify_peer, cacerts: [cert]]]) +``` diff --git a/README.md b/README.md index 9472be3f..c796fe1a 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ [v0.24]: https://github.com/Strech/avrora/releases/tag/v0.24.0 [v0.25]: https://github.com/Strech/avrora/releases/tag/v0.25.0 [v0.26]: https://github.com/Strech/avrora/releases/tag/v0.26.0 +[v0.28]: https://github.com/Strech/avrora/releases/tag/v0.28.0 [1]: https://avro.apache.org/ [2]: https://www.confluent.io/confluent-schema-registry [3]: https://docs.confluent.io/current/schema-registry/serializer-formatter.html#wire-format @@ -34,6 +35,7 @@ [9]: https://github.com/Strech/avrora/wiki/Schema-name-resolution [10]: https://github.com/Strech/avrora/pull/70 [11]: https://github.com/klarna/erlavro#decoder-hooks +[12]: https://www.erlang.org/docs/26/man/ssl#type-client_cacerts # Getting Started @@ -62,7 +64,7 @@ Add Avrora to `mix.exs` as a dependency ```elixir def deps do [ - {:avrora, "~> 0.21"} + {:avrora, "~> 0.27"} ] end ``` @@ -83,10 +85,12 @@ Create your private Avrora client module defmodule MyClient do use Avrora.Client, otp_app: :my_application, + schemas_path: "./priv/schemas", registry_url: "http://localhost:8081", registry_auth: {:basic, ["username", "password"]}, registry_user_agent: "Avrora/0.25.0 Elixir", - schemas_path: "./priv/schemas", + registry_ssl_cacerts: File.read!("./priv/trusted.der"), + registry_ssl_cacert_path: "./priv/trusted.crt", registry_schemas_autoreg: false, convert_null_values: false, convert_map_to_proplist: false, @@ -104,10 +108,12 @@ Configure the `Avrora` shared client in `config/config.exs` ```elixir config :avrora, otp_app: :my_application, # optional, if you want to use it as a root folder for `schemas_path` + schemas_path: "./priv/schemas", registry_url: "http://localhost:8081", registry_auth: {:basic, ["username", "password"]}, # optional registry_user_agent: "Avrora/0.24.2 Elixir", # optional: if you want to return previous behaviour, set it to `nil` - schemas_path: "./priv/schemas", + registry_ssl_cacerts: File.read!("./priv/trusted.der"), # optional: if you have DER-encoded certificate + registry_ssl_cacert_path: "./priv/trusted.crt", # optional: if you have PEM-encoded certificate file registry_schemas_autoreg: false, # optional: if you want manually register schemas convert_null_values: false, # optional: if you want to keep decoded `:null` values as is convert_map_to_proplist: false, # optional: if you want to restore the old behavior for decoding map-type @@ -116,10 +122,12 @@ config :avrora, ``` - `otp_app`[v0.22] - Name of the OTP application to use for runtime configuration via env, default `nil` +- `schemas_path` - Base path for locally stored schema files, default `./priv/schemas` - `registry_url` - URL for the Schema Registry, default `nil` - `registry_auth` – Credentials to authenticate in the Schema Registry, default `nil` - `registry_user_agent`[v0.25] - HTTP `User-Agent` header for Schema Registry requests, default `Avrora/ Elixir` -- `schemas_path` - Base path for locally stored schema files, default `./priv/schemas` +- `registry_ssl_cacerts`[v0.28] - DER-encoded certificates, but [without combined support][12], default `nil` +- `registry_ssl_cacert_path`[v0.28] - Path to a file containing PEM-encoded CA certificates, default `nil` - `registry_schemas_autoreg`[v0.13] - Flag for automatic schemas registration in the Schema Registry, default `true` - `convert_null_values`[v0.14] - Flag for automatic conversion of decoded `:null` values into `nil`, default `true` - `convert_map_to_proplist`[v0.15] restore old behaviour and confiugre decoding map-type to proplist, default `false` @@ -146,6 +154,9 @@ recommended to set `otp_app` which will point to your OTP applications. This wil to have a per-client runtime resolution for all configuration options (i.e. `schemas_path`) with a fallback to staticly defined in a client itself.[v0.23] +:bulb: If both `registry_ssl_cacerts` and `registry_ssl_cacert_path` given, then +`registry_ssl_cacerts` has a priority. + ## Start cache process Avrora uses an in-memory cache to speed up schema lookup. diff --git a/lib/avrora/client.ex b/lib/avrora/client.ex index 5a45f307..ca89b7f7 100644 --- a/lib/avrora/client.ex +++ b/lib/avrora/client.ex @@ -109,9 +109,16 @@ defmodule Avrora.Client do if is_nil(@otp_app), do: Path.expand(path), else: Application.app_dir(@otp_app, path) end + def registry_ssl_cacert_path do + path = get(@opts, :registry_ssl_cacert_path, nil) + + if is_nil(path), do: nil, else: Path.expand(path) + end + def registry_url, do: get(@opts, :registry_url, nil) def registry_auth, do: get(@opts, :registry_auth, nil) def registry_user_agent, do: get(@opts, :registry_user_agent, "Avrora/#{version()} Elixir") + def registry_ssl_cacerts, do: get(@opts, :registry_ssl_cacerts, nil) def registry_schemas_autoreg, do: get(@opts, :registry_schemas_autoreg, true) def convert_null_values, do: get(@opts, :convert_null_values, true) def convert_map_to_proplist, do: get(@opts, :convert_map_to_proplist, false) diff --git a/lib/avrora/config.ex b/lib/avrora/config.ex index 8c120ed1..459bf9ff 100644 --- a/lib/avrora/config.ex +++ b/lib/avrora/config.ex @@ -9,6 +9,8 @@ defmodule Avrora.Config do * `registry_url` URL for Schema Registry, default `nil` * `registry_auth` authentication settings for Schema Registry, default `nil` * `registry_user_agent` HTTP `User-Agent` header for Schema Registry requests, default `Avrora/ Elixir` + * `registry_ssl_cacerts` DER-encoded trusted certificate (not combined) (see https://www.erlang.org/docs/26/man/ssl#type-client_cacerts), default `nil` + * `registry_ssl_cacert_path` path to a file containing PEM-encoded CA certificates, default `nil` * `registry_schemas_autoreg` automatically register schemas in Schema Registry, default `true` * `convert_null_values` convert `:null` values in the decoded message into `nil`, default `true` * `convert_map_to_proplist` bring back old behavior and configure decoding AVRO map-type as proplist, default `false` @@ -28,6 +30,8 @@ defmodule Avrora.Config do @callback registry_url :: String.t() | nil @callback registry_auth :: tuple() | nil @callback registry_user_agent :: String.t() | nil + @callback registry_ssl_cacerts :: binary() | nil + @callback registry_ssl_cacert_path :: String.t() | nil @callback registry_schemas_autoreg :: boolean() @callback convert_null_values :: boolean() @callback convert_map_to_proplist :: boolean() @@ -56,6 +60,18 @@ defmodule Avrora.Config do @doc false def registry_user_agent, do: get_env(:registry_user_agent, "Avrora/#{version()} Elixir") + # NOTE: Starting OTP-25 it is possible to call `public_key:cacerts_get` + # See https://erlef.github.io/security-wg/secure_coding_and_deployment_hardening/ssl + @doc false + def registry_ssl_cacerts, do: get_env(:registry_ssl_cacerts, nil) + + @doc false + def registry_ssl_cacert_path do + path = get_env(:registry_ssl_cacert_path, nil) + + if is_nil(path), do: nil, else: Path.expand(path) + end + @doc false def registry_schemas_autoreg, do: get_env(:registry_schemas_autoreg, true) diff --git a/lib/avrora/http_client.ex b/lib/avrora/http_client.ex index c9a6826b..43c62122 100644 --- a/lib/avrora/http_client.ex +++ b/lib/avrora/http_client.ex @@ -5,12 +5,14 @@ defmodule Avrora.HTTPClient do @callback get(String.t(), keyword(String.t())) :: {:ok, map()} | {:error, term()} @callback post(String.t(), String.t(), keyword(String.t())) :: {:ok, map()} | {:error, term()} + @default_ssl_options [verify: :verify_none] @doc false @spec get(String.t(), keyword(String.t())) :: {:ok, map()} | {:error, term()} def get(url, options \\ []) do with {:ok, headers} <- extract_headers(options), - {:ok, {{_, status, _}, _, body}} <- :httpc.request(:get, {'#{url}', headers}, [ssl: ssl_options()], []) do + {:ok, ssl_options} <- extract_ssl(options), + {:ok, {{_, status, _}, _, body}} <- :httpc.request(:get, {'#{url}', headers}, [ssl: ssl_options], []) do handle(status, body) end end @@ -21,8 +23,9 @@ defmodule Avrora.HTTPClient do with {:ok, body} <- Jason.encode(%{"schema" => payload}), {:ok, content_type} <- Keyword.fetch(options, :content_type), {:ok, headers} <- extract_headers(options), + {:ok, ssl_options} <- extract_ssl(options), {:ok, {{_, status, _}, _, body}} <- - :httpc.request(:post, {'#{url}', headers, [content_type], body}, [ssl: ssl_options()], []) do + :httpc.request(:post, {'#{url}', headers, [content_type], body}, [ssl: ssl_options], []) do handle(status, body) end end @@ -40,6 +43,7 @@ defmodule Avrora.HTTPClient do end end + defp extract_ssl(options), do: {:ok, Keyword.get(options, :ssl_options, @default_ssl_options)} defp handle(200 = _status, body), do: Jason.decode(body) defp handle(status, body) do @@ -48,6 +52,4 @@ defmodule Avrora.HTTPClient do {:error, _} -> {:error, {status, body}} end end - - defp ssl_options, do: [verify: :verify_none] end diff --git a/lib/avrora/storage/registry.ex b/lib/avrora/storage/registry.ex index fac20558..3349d6ea 100644 --- a/lib/avrora/storage/registry.ex +++ b/lib/avrora/storage/registry.ex @@ -116,20 +116,31 @@ defmodule Avrora.Storage.Registry do defp http_client_get(path) do if configured?(), - do: path |> to_url() |> http_client().get(headers()) |> handle(), + do: path |> to_url() |> http_client().get(options()) |> handle(), else: {:error, :unconfigured_registry_url} end defp http_client_post(path, payload) do if configured?() do - headers = headers() |> Keyword.put(:content_type, @content_type) - path |> to_url() |> http_client().post(payload, headers) |> handle() + options = [{:content_type, @content_type} | options()] + path |> to_url() |> http_client().post(payload, options) |> handle() else {:error, :unconfigured_registry_url} end end - # NOTE: Maybe move to compile-time? + # NOTE: Maybe move to compile-time somehow? + defp options do + ssl_options = + cond do + !is_nil(registry_ssl_cacerts()) -> [verify: :verify_peer, cacerts: [registry_ssl_cacerts()]] + !is_nil(registry_ssl_cacert_path()) -> [verify: :verify_peer, cacertfile: registry_ssl_cacert_path()] + true -> [verify: :verify_none] + end + + [{:ssl_options, ssl_options} | headers()] + end + defp headers do authorization = case registry_auth() do @@ -143,7 +154,7 @@ defmodule Avrora.Storage.Registry do case registry_user_agent() do nil -> authorization - user_agent -> authorization |> Keyword.put(:user_agent, user_agent) + user_agent -> [{:user_agent, user_agent} | authorization] end end @@ -169,4 +180,6 @@ defmodule Avrora.Storage.Registry do defp registry_url, do: Config.self().registry_url() defp registry_auth, do: Config.self().registry_auth() defp registry_user_agent, do: Config.self().registry_user_agent() + defp registry_ssl_cacerts, do: Config.self().registry_ssl_cacerts() + defp registry_ssl_cacert_path, do: Config.self().registry_ssl_cacert_path() end diff --git a/mix.exs b/mix.exs index 5700be89..ea7dc7b7 100644 --- a/mix.exs +++ b/mix.exs @@ -1,7 +1,7 @@ defmodule Avrora.MixProject do use Mix.Project - @version "0.27.0" + @version "0.28.0" def project do [ diff --git a/test/avrora/storage/registry_test.exs b/test/avrora/storage/registry_test.exs index 061c1de5..97566f23 100644 --- a/test/avrora/storage/registry_test.exs +++ b/test/avrora/storage/registry_test.exs @@ -18,9 +18,8 @@ defmodule Avrora.Storage.RegistryTest do describe "get/1" do test "when request by subject name of schema with reference without version was successful" do Avrora.HTTPClientMock - |> expect(:get, fn url, options -> + |> expect(:get, fn url, _ -> assert url == "http://reg.loc/subjects/io.confluent.Account/versions/latest" - assert options == [] { :ok, @@ -37,9 +36,8 @@ defmodule Avrora.Storage.RegistryTest do end) Avrora.HTTPClientMock - |> expect(:get, fn url, options -> + |> expect(:get, fn url, _ -> assert url == "http://reg.loc/subjects/io.confluent.User/versions/1" - assert options == [] { :ok, @@ -62,9 +60,8 @@ defmodule Avrora.Storage.RegistryTest do test "when request by subject name of schema with reference was unsuccessful because of reference schema not found" do Avrora.HTTPClientMock - |> expect(:get, fn url, options -> + |> expect(:get, fn url, _ -> assert url == "http://reg.loc/subjects/io.confluent.Account/versions/latest" - assert options == [] { :ok, @@ -84,9 +81,8 @@ defmodule Avrora.Storage.RegistryTest do end) Avrora.HTTPClientMock - |> expect(:get, fn url, options -> + |> expect(:get, fn url, _ -> assert url == "http://reg.loc/subjects/io.confluent.Unexisting/versions/latest" - assert options == [] {:error, subject_not_found_parsed_error()} end) @@ -96,9 +92,8 @@ defmodule Avrora.Storage.RegistryTest do test "when request by subject name without version was successful" do Avrora.HTTPClientMock - |> expect(:get, fn url, options -> + |> expect(:get, fn url, _ -> assert url == "http://reg.loc/subjects/io.confluent.Payment/versions/latest" - assert options == [] { :ok, @@ -120,9 +115,8 @@ defmodule Avrora.Storage.RegistryTest do test "when request by subject name with version was successful" do Avrora.HTTPClientMock - |> expect(:get, fn url, options -> + |> expect(:get, fn url, _ -> assert url == "http://reg.loc/subjects/io.confluent.Payment/versions/10" - assert options == [] { :ok, @@ -144,9 +138,8 @@ defmodule Avrora.Storage.RegistryTest do test "when request by subject name was unsuccessful" do Avrora.HTTPClientMock - |> expect(:get, fn url, options -> + |> expect(:get, fn url, _ -> assert url == "http://reg.loc/subjects/io.confluent.Payment/versions/latest" - assert options == [] {:error, subject_not_found_parsed_error()} end) @@ -156,9 +149,8 @@ defmodule Avrora.Storage.RegistryTest do test "when request by global ID was successful" do Avrora.HTTPClientMock - |> expect(:get, fn url, options -> + |> expect(:get, fn url, _ -> assert url == "http://reg.loc/schemas/ids/1" - assert options == [] {:ok, %{"schema" => json_schema()}} end) @@ -176,7 +168,7 @@ defmodule Avrora.Storage.RegistryTest do Avrora.HTTPClientMock |> expect(:get, fn url, options -> assert url == "http://reg.loc/schemas/ids/1" - assert options == [authorization: "Basic bG9naW46cGFzc3dvcmQ="] + assert Keyword.fetch!(options, :authorization) == "Basic bG9naW46cGFzc3dvcmQ=" {:ok, %{"schema" => json_schema()}} end) @@ -190,9 +182,8 @@ defmodule Avrora.Storage.RegistryTest do test "when request by global ID was unsuccessful" do Avrora.HTTPClientMock - |> expect(:get, fn url, options -> + |> expect(:get, fn url, _ -> assert url == "http://reg.loc/schemas/ids/1" - assert options == [] {:error, version_not_found_parsed_error()} end) @@ -202,9 +193,8 @@ defmodule Avrora.Storage.RegistryTest do test "when request by global ID with reference was successful" do Avrora.HTTPClientMock - |> expect(:get, fn url, options -> + |> expect(:get, fn url, _ -> assert url == "http://reg.loc/schemas/ids/43" - assert options == [] { :ok, @@ -221,9 +211,8 @@ defmodule Avrora.Storage.RegistryTest do end) Avrora.HTTPClientMock - |> expect(:get, fn url, options -> + |> expect(:get, fn url, _ -> assert url == "http://reg.loc/subjects/io.confluent.User/versions/1" - assert options == [] { :ok, @@ -244,66 +233,66 @@ defmodule Avrora.Storage.RegistryTest do assert schema.json == json_schema_with_reference_denormalized() end - test "when registry url is unconfigured" do - stub(Avrora.ConfigMock, :registry_url, fn -> nil end) - - assert Registry.get("anything") == {:error, :unconfigured_registry_url} - end - end - - describe "put/2" do - test "when request was successful" do + test "when request should not perform SSL verification" do Avrora.HTTPClientMock - |> expect(:post, fn url, payload, options -> - assert url == "http://reg.loc/subjects/io.confluent.Payment/versions" - assert payload == json_schema() - assert options == [content_type: "application/vnd.schemaregistry.v1+json"] + |> expect(:get, fn url, options -> + assert url == "http://reg.loc/schemas/ids/1" + assert Keyword.fetch!(options, :ssl_options) == [verify: :verify_none] - {:ok, %{"id" => 1}} + {:ok, %{"schema" => json_schema()}} end) - {:ok, schema} = Registry.put("io.confluent.Payment", json_schema()) + {:ok, schema} = Registry.get(1) assert schema.id == 1 assert is_nil(schema.version) assert schema.full_name == "io.confluent.Payment" end - test "when request with basic auth was successful" do - stub(Avrora.ConfigMock, :registry_auth, fn -> {:basic, ["login", "password"]} end) + test "when request should not perform SSL verification based on given cert" do + stub(Avrora.ConfigMock, :registry_ssl_cacerts, fn -> <<48, 130, 3, 201>> end) + stub(Avrora.ConfigMock, :registry_ssl_cacert_path, fn -> "path/to/file" end) Avrora.HTTPClientMock - |> expect(:post, fn url, payload, options -> - assert url == "http://reg.loc/subjects/io.confluent.Payment/versions" - assert payload == json_schema() - - assert options == [ - content_type: "application/vnd.schemaregistry.v1+json", - authorization: "Basic bG9naW46cGFzc3dvcmQ=" - ] + |> expect(:get, fn url, options -> + assert url == "http://reg.loc/schemas/ids/1" + assert Keyword.fetch!(options, :ssl_options) == [verify: :verify_peer, cacerts: [<<48, 130, 3, 201>>]] - {:ok, %{"id" => 1}} + {:ok, %{"schema" => json_schema()}} end) - {:ok, schema} = Registry.put("io.confluent.Payment", json_schema()) + assert :ok == Registry.get(1) |> elem(0) + end - assert schema.id == 1 - assert is_nil(schema.version) - assert schema.full_name == "io.confluent.Payment" + test "when request should not perform SSL verification based on given cert file" do + stub(Avrora.ConfigMock, :registry_ssl_cacert_path, fn -> "path/to/file" end) + + Avrora.HTTPClientMock + |> expect(:get, fn url, options -> + assert url == "http://reg.loc/schemas/ids/1" + assert Keyword.fetch!(options, :ssl_options) == [verify: :verify_peer, cacertfile: "path/to/file"] + + {:ok, %{"schema" => json_schema()}} + end) + + assert :ok == Registry.get(1) |> elem(0) end - test "when request with user agent was successful" do - stub(Avrora.ConfigMock, :registry_user_agent, fn -> "Avrora/0.0.1 Elixir" end) + test "when registry url is unconfigured" do + stub(Avrora.ConfigMock, :registry_url, fn -> nil end) + assert Registry.get("anything") == {:error, :unconfigured_registry_url} + end + end + + describe "put/2" do + test "when request was successful" do Avrora.HTTPClientMock |> expect(:post, fn url, payload, options -> assert url == "http://reg.loc/subjects/io.confluent.Payment/versions" assert payload == json_schema() - - assert options == [ - content_type: "application/vnd.schemaregistry.v1+json", - user_agent: "Avrora/0.0.1 Elixir" - ] + assert Keyword.fetch!(options, :content_type) == "application/vnd.schemaregistry.v1+json" + assert Keyword.fetch!(options, :ssl_options) == [verify: :verify_none] {:ok, %{"id" => 1}} end) @@ -348,6 +337,67 @@ defmodule Avrora.Storage.RegistryTest do assert Registry.put("io.confluent.Payment", ~s({"type":"string"})) == {:error, :conflict} end + test "when request should send Authorization header" do + stub(Avrora.ConfigMock, :registry_auth, fn -> {:basic, ["login", "password"]} end) + + Avrora.HTTPClientMock + |> expect(:post, fn url, payload, options -> + assert url == "http://reg.loc/subjects/io.confluent.Payment/versions" + assert payload == json_schema() + assert Keyword.fetch!(options, :authorization) == "Basic bG9naW46cGFzc3dvcmQ=" + + {:ok, %{"id" => 1}} + end) + + assert :ok == Registry.put("io.confluent.Payment", json_schema()) |> elem(0) + end + + test "when request should send User-Agent header" do + stub(Avrora.ConfigMock, :registry_user_agent, fn -> "Avrora/0.0.1 Elixir" end) + + Avrora.HTTPClientMock + |> expect(:post, fn url, payload, options -> + assert url == "http://reg.loc/subjects/io.confluent.Payment/versions" + assert payload == json_schema() + assert Keyword.fetch!(options, :user_agent) == "Avrora/0.0.1 Elixir" + + {:ok, %{"id" => 1}} + end) + + assert :ok == Registry.put("io.confluent.Payment", json_schema()) |> elem(0) + end + + test "when request should not perform SSL verification based on given cert" do + stub(Avrora.ConfigMock, :registry_ssl_cacerts, fn -> <<48, 130, 3, 201>> end) + stub(Avrora.ConfigMock, :registry_ssl_cacert_path, fn -> "path/to/file" end) + + Avrora.HTTPClientMock + |> expect(:post, fn url, payload, options -> + assert url == "http://reg.loc/subjects/io.confluent.Payment/versions" + assert payload == json_schema() + assert Keyword.fetch!(options, :ssl_options) == [verify: :verify_peer, cacerts: [<<48, 130, 3, 201>>]] + + {:ok, %{"id" => 1}} + end) + + assert :ok == Registry.put("io.confluent.Payment", json_schema()) |> elem(0) + end + + test "when request should not perform SSL verification based on given cert file" do + stub(Avrora.ConfigMock, :registry_ssl_cacert_path, fn -> "path/to/file" end) + + Avrora.HTTPClientMock + |> expect(:post, fn url, payload, options -> + assert url == "http://reg.loc/subjects/io.confluent.Payment/versions" + assert payload == json_schema() + assert Keyword.fetch!(options, :ssl_options) == [verify: :verify_peer, cacertfile: "path/to/file"] + + {:ok, %{"id" => 1}} + end) + + assert :ok == Registry.put("io.confluent.Payment", json_schema()) |> elem(0) + end + test "when registry url is unconfigured" do stub(Avrora.ConfigMock, :registry_url, fn -> nil end) diff --git a/test/support/config.ex b/test/support/config.ex index 184431f3..f674f551 100644 --- a/test/support/config.ex +++ b/test/support/config.ex @@ -42,6 +42,10 @@ defmodule Support.Config do @impl true def registry_user_agent, do: nil @impl true + def registry_ssl_cacerts, do: nil + @impl true + def registry_ssl_cacert_path, do: nil + @impl true def registry_schemas_autoreg, do: true @impl true def convert_null_values, do: true