Pillar

Clickhouse elixir driver via HTTP connection

Features

• Direct Usage with connection structure
• Pool of workers
• Async insert
• Buffer for periodical bulk inserts
• Migrations

Usage

Direct Usage with connection structure

conn = Pillar.Connection.new("http://user:password@localhost:8123/database")

# params are passed in brackets {} in sql query, and map strtucture does fill query by values
sql = "SELECT count(*) FROM users WHERE lastname = {lastname}"

params = %{lastname: "Smith"}

{:ok, result} = Pillar.query(conn, sql, params)

result 
#=> [%{"count(*)" => 347}]

Pool of workers

Recommended usage, because of limited connections and supervised workers

  defmodule ClickhouseMaster do
      use Pillar, 
        connection_strings: [
          "http://user:password@host-master-1:8123/database",
          "http://user:password@host-master-2:8123/database"
        ],
        name: __MODULE__,
        pool_size: 15
  end

  ClickhouseMaster.start_link()

  {:ok, result} = ClickhouseMaster.select(sql, %{param: value})

Async insert

  connection = Pillar.Connection.new("http://user:password@host-master-1:8123/database")

  Pillar.async_insert(connection, "INSERT INTO events (user_id, event) SELECT {user_id}, {event}", %{
    user_id: user.id,
    event: "password_changed"
  }) # => :ok

Buffer for periodical bulk inserts

For this feature required Pool of workers

  defmodule BulkToLogs do
    use Pillar.BulkInsertBuffer,
      pool: ClickhouseMaster,
      table_name: "logs",
      interval_between_inserts_in_seconds: 5
  end

:ok = BulkToLogs.insert(%{value: "online", count: 133, datetime: DateTime.utc_now()})
:ok = BulkToLogs.insert(%{value: "online", count: 134, datetime: DateTime.utc_now()})
:ok = BulkToLogs.insert(%{value: "online", count: 132, datetime: DateTime.utc_now()})
....

# all this records will be inserted with 5 second interval

Migrations

Migrations can be generated with mix task mix pillar.gen.migration migration_name

mix pillar.gen.migration events_table

But for launching them we have to write own task, like this:

defmodule Mix.Tasks.MigrateClickhouse do
  use Mix.Task
  def run(_args) do
    connection_string = Application.get_env(:my_project, :clickhouse_url)  
    conn = Pillar.Connection.new(connection_string)
    Pillar.Migrations.migrate(conn)
  end
end

And launch this via command

mix migrate_clickhouse

Contribution

Feel free to make a pull request. All contributions are appreciated!