Merge pull request #22 from idealista/develop

New release

CHANGELOG.md

@@ -5,6 +5,28 @@ This project adheres to [Semantic Versioning](http://semver.org/) and [Keep a ch
 
 ## [Unreleased](https://github.com/idealista/clickhouse_role/tree/develop)
 
+## [3.1.0](https://github.com/idealista/clickhouse_role/tree/3.1.0) (2022-02-14)
+
+### :heavy_plus_sign: Added
+
+- Option to use custom config xml files
+- [#18](https://github.com/idealista/clickhouse_role/issues/18) Option to create and remove basic users, roles, profiles, quotas, etc @ultraheroe
+
+### :repeat: Updated
+
+- Default ClickHouse version.
+- Requirements versions.
+- Readme instructions.
+- Molecule verify update to use checksum instead of the deprecated command.
+- Default molecule scenario and tests
+- Tasks orders and role files skeleton
+- [#20](https://github.com/idealista/clickhouse_role/issues/20) clickhouse_macros var refactor @ultraheroe
+
+### :hammer_and_wrench: Fixed
+
+- [#16](https://github.com/idealista/clickhouse_role/issues/16) Fix missing user, password and compression values in remote_servers @ultraheroe
+- [#14](https://github.com/idealista/clickhouse_role/issues/14) Wait for service listening @jmonterrubio
+
 ## [3.0.1](https://github.com/idealista/clickhouse_role/tree/3.0.1) (2021-12-14)
 
 ### :hammer_and_wrench: Fixed

Pipfile

@@ -6,11 +6,11 @@ verify_ssl = true
 [dev-packages]
 
 [packages]
-ansible = "==4.6.0"
+ansible = "==4.10.0"
 molecule = "==3.5.2"
 docker = "==5.0.3"
-ansible-lint = "==5.2.1"
-molecule-containers = "==1.0.0"
+ansible-lint = "==5.3.2"
+molecule-containers = "==1.0.2"
 yamllint = "==1.26.3"
 
 [requires]

README.md

@@ -1,10 +1,10 @@
 # Clickhouse Ansible role
+
 ![GitHub release (latest by date)](https://img.shields.io/github/v/release/idealista/clickhouse_role?color=%23B62682)
 [![Build Status](https://travis-ci.org/idealista/clickhouse_role.png)](https://travis-ci.org/idealista/clickhouse_role) [![Ansible Galaxy](https://img.shields.io/badge/galaxy-idealista.clickhouse_role-B62682.svg)](https://galaxy.ansible.com/idealista/clickhouse_role)
 
 ![Logo](https://raw.githubusercontent.com/idealista/clickhouse_role/main/logo.gif)
 
-
 This ansible role installs [Clickhouse](https://clickhouse.com/) in a Debian environment. It has been tested for Debian bullseye and buster.
 
 This role has been generated using the [cookiecutter](https://github.com/cookiecutter/cookiecutter) tool, you can generate a similar role that fits your needs using the this [cookiecutter template](https://github.com/idealista/cookiecutter-ansible-role).
@@ -21,15 +21,16 @@ This role has been generated using the [cookiecutter](https://github.com/cookiec
 - [Contributing](#contributing-construction_worker)
 
 ## Getting Started :checkered_flag:
+
 These instructions will get you a copy of the role for your Ansible playbook. Once launched, it will install [Clickhouse](https://clickhouse.com/) in a Debian system.
 
 ### Prerequisites :ballot_box_with_check:
 
-Ansible 4.6.x version installed.
+Ansible 4.x.x version installed.
 
 Molecule 3.x.x version installed.
 
-For testing purposes, [Molecule](https://molecule.readthedocs.io/) with [Docker](https://www.docker.com/) as driver and  [Goss] (https://github.com/aelsabbahy/goss) as verifier.
+For testing purposes, [Molecule](https://molecule.readthedocs.io/) with [Docker](https://www.docker.com/) as driver and [Goss](https://github.com/aelsabbahy/goss) as verifier.
 
 ### Installing :inbox_tray:
 
@@ -63,6 +64,49 @@ Look to the [defaults](defaults/main.yml) properties file to see the possible co
 
 - [`main.yml`](./defaults/main.yml) for superset general purpose vars.
 
+### 👉 Don't forget
+
+- 🦸 To set your Admin user and use a secure 🔑 password.
+- 📝 To set the `clickhouse_custom_config_file_path` and / or `clickhouse_custom_users_file_path` if you are going to use custom config files.
+  - 👉 See the [ClickHouse doc](https://clickhouse.com/docs/en/operations/configuration-files/).
+- ☑️ To enable or disable using `clickhouse_role_manage_X` vars what things the role should manage.
+- 📝 To set users, quotas, profiles, databases to create.
+  - ℹ️ Or to unset if you want to DROP things.
+  - 👉 See the default molecule scenario [`group_vars`](./molecule/default/group_vars/clickhouse_group.yml) for more
+
+### ❗ You must know
+
+- ⚠️ `clickhouse_replicated_tables_macros` is deprecated, please use `clickhouse_macros` var
+- ⚠️ Note that are two ways to set users for ClickHouse, `users.xml` or via SQL-query, to distinguish both methods note that in this role we use `clickhouse_custom_users_xml` and `clickhouse_custom_users` respectively (SQL recommended).
+- ❗ To make us of the 'EXCEPT' clauses for [quota assignation](https://clickhouse.com/docs/en/sql-reference/statements/create/quota/) or [user grantees](https://clickhouse.com/docs/en/sql-reference/statements/create/user/#grantees) for example, you can add a minus or dash _( - )_ before the name.
+- ❗ When setting `password_type` for users, it should be one of [this](https://clickhouse.com/docs/en/sql-reference/statements/create/user/#identification)
+- ❗ When setting `keyed` for quota, it should be one of [this](https://clickhouse.com/docs/en/sql-reference/statements/create/quota/)
+- ❗ In case you're using LDAP or Kerberos, set each with their own property `ldap_server` or 'kerberos' so `password_type` is not required
+
+##### Custom user definition example:
+
+  ```yml
+  clickhouse_custom_users:
+    - user:
+      name: "Takumi"
+      password_type: plaintext_password
+      password: "AE86"
+      networks:
+        - "IP '::/0'"
+      settings:
+        - "max_memory_usage = 10000000000"
+      role:
+        - projectd_members
+        - tofu_shop
+      profile:
+        - default
+      grantees:
+        - projectd_members
+      quota: "default"
+      databases: [ProjectD]
+      # ldap_server: project.d
+      # kerberos: ""
+
 ## Testing :test_tube:
 
 ### Install dependencies
@@ -81,15 +125,9 @@ $ pipenv run molecule test
 
 ## Built With :building_construction:
 
-ansible==4.6.0
-ansible-lint==5.2.1
-molecule==3.5.2
-docker==5.0.3
-molecule-containers==1.0.0
-yamllint==1.26.3
-
-![Ansible](https://img.shields.io/badge/ansible-4.6.0-green.svg)
+![Ansible](https://img.shields.io/badge/ansible-4.10.0-green.svg)
 ![Molecule](https://img.shields.io/badge/molecule-3.5.2-green.svg)
+![Docker](https://img.shields.io/badge/docker-5.0.3-blue.svg)
 ![Goss](https://img.shields.io/badge/goss-0.3.13-green.svg)
 
 ## Versioning :card_file_box:
@@ -100,7 +138,7 @@ Additionaly you can see what change in each version in the [CHANGELOG.md](CHANGE
 
 ## Authors :superhero:
 
-* **Idealista** - *Work with* - [idealista](https://github.com/idealista)
+- **Idealista** - _Work with_ - [idealista](https://github.com/idealista)
 
 See also the list of [contributors](https://github.com/idealista/clickhouse_role/contributors) who participated in this project.
 

defaults/main.yml

@@ -2,9 +2,16 @@
 
 ## General
 # Version
-clickhouse_version: 21.11.5.33
+clickhouse_version: 22.1.3.7
 clickhouse_release_type: stable  # Values should be: stable, lts or testing
 
+# Clickhouse role management
+clickhouse_role_manage_users: False
+clickhouse_role_manage_roles: False
+clickhouse_role_manage_databases: False
+clickhouse_role_manage_settings_profiles: False
+clickhouse_role_manage_quotas: False
+
 ## Service options
 # Documentation
 clickhouse_documentation_link: https://clickhouse.tech/docs/en/
@@ -49,17 +56,16 @@ clickhouse_packages:
   - "clickhouse-server={{clickhouse_version}}"
 
 # Config templates
-clickhouse_config_files:
-  - "{{ clickhouse_config_file_name }}"
-  - "{{ clickhouse_users_file_name }}"
-
 clickhouse_service_template_path: clickhouse-server.service.j2
 clickhouse_config_file_name: config.xml
 clickhouse_users_file_name: users.xml
+
 # Directories
 clickhouse_base_path: "/var/lib"
 clickhouse_config_directory: "/etc/clickhouse-server"
-clickhouse_config_directory_mode: 0760
+clickhouse_config_directory_mode: 0700
+clickhouse_config_directory_config_d: "{{ clickhouse_config_directory }}/config.d"
+clickhouse_config_directory_users_d: "{{ clickhouse_config_directory }}/users.d"
 clickhouse_log_directory: "/var/log/clickhouse-server"
 clickhouse_log_directory_mode: 0740
 clickhouse_data_directory: "{{ clickhouse_base_path }}/clickhouse"
@@ -68,6 +74,8 @@ clickhouse_tmp_directory: "{{ clickhouse_base_path }}/clickhouse/tmp"
 clickhouse_tmp_directory_mode: 0755
 clickhouse_user_files_directory: "{{ clickhouse_base_path }}/clickhouse/user_files"
 clickhouse_access_control_directory: "{{ clickhouse_base_path }}/clickhouse/access/"
+# clickhouse_custom_config_file_path: "{{ playbook_dir }}/config_files/config"
+# clickhouse_custom_users_file_path: "{{ playbook_dir }}/config_files/users"
 
 # Logs
 clickhouse_logger:
@@ -364,12 +372,23 @@ clickhouse_remote_servers:
 #     host:
 #     port:
 
-# clickhouse_replicated_tables_macros:
+# clickhouse_replicated_tables_macros:  # Var deprecated use clickhouse_macros
 #   - server: servername
 #     macro: |
 #       <shard>1</shard>
 #       <replica>1</replica>
 
+# Macros
+# clickhouse_macros:
+#   - macro: |
+#       <shard>1</shard>
+#       <replica>1</replica>
+#     server:
+#       - clickhouse1
+#       - clickhouse2
+#   - macro: |
+#       <db_default>default</db_default>
+
 # Graphite
 # clickhouse_graphite:
 #   - node:
@@ -608,13 +627,22 @@ clickhouse_default_users:
     profile: readonly
     quota: default
 
+# clickhouse_admin_users:
+#   - user:
+#     name: "{{ clickhouse_admin_user }}"
+#     password: "{{ clickhouse_admin_password }}"
+#     networks: "{{ clickhouse_default_networks }}"
+#     profile: "default"
+#     quota: "default"
+#     access_management: 1
+
 # Users examples
 # clickhouse_admin_user: admin
 # clickhouse_admin_password: admin
 
 ## See templates/users.xml.j2 for other possible passwords format (plain text is not recommended).
 
-# clickhouse_custom_users:
+# clickhouse_custom_users_xml:
 #   - user:
 #     name: "{{ clickhouse_admin_user }}"
 #     password: "{{ clickhouse_admin_password }}"
@@ -629,14 +657,21 @@ clickhouse_default_users:
 #     profile: "default"
 #     quota: "default"
 #     databases: [test1]
+
+clickhouse_custom_users: []
+# clickhouse_custom_users:
 #   - user:
 #     name: "user2"
+#     password_type: PLAINTEXT_PASSWORD  # one of: NO_PASSWORD, PLAINTEXT_PASSWORD, SHA256_PASSWORD, DOUBLE_SHA1_PASSWORD, LDAP, KERBEROS, SHA256_HASH, DOUBLE_SHA1_HASH
 #     password: "user2"
 #     networks: "{{ clickhouse_default_networks }}"
 #     profile: "default"
 #     quota: "default"
 #     databases: [test2]
 
+# Roles
+clickhouse_custom_roles: []
+
 # Quotas
 clickhouse_default_quotas:
   - quota:
@@ -648,9 +683,14 @@ clickhouse_default_quotas:
     read_rows: 0
     execution_time: 0gv
 
-# clickhouse_custom_quotas: []
+# clickhouse_custom_quotas_xml: []
+clickhouse_custom_quotas: []
+
+# Settings profiles
+clickhouse_custom_settings_profiles: []
 
 # Databases examples
+clickhouse_custom_databases: []
 # clickhouse_custom_databases:
 #   - name: test1
 #   - name: test2