diff --git a/ansible-scylla-node/defaults/main.yml b/ansible-scylla-node/defaults/main.yml index d062a67d..440755a4 100644 --- a/ansible-scylla-node/defaults/main.yml +++ b/ansible-scylla-node/defaults/main.yml @@ -89,7 +89,7 @@ install_only: false # Change here or override in vars scylla_nic: "{{ ansible_default_ipv4.interface }}" -cluster_local_files_path: "{{ inventory_dir }}" +manager_local_files_path: "{{ playbook_dir }}" # online|offline - offline will disable all the attepts to download packages # node_exporter will need to be installed manually @@ -297,7 +297,7 @@ scylla_ssl: # Note that by default the {{ inventory_hostname }} is part of the localhost_system_key_directory. This allows you to define # different keys per host. # e.g.: -# If your localhost_system_key_directory is set to "{{ inventory_dir }}/encryption_at_rest/{{ inventory_hostname }}/system_encryption_keys" +# If your localhost_system_key_directory is set to "{{ playbook_dir }}/encryption_at_rest/{{ inventory_hostname }}/system_encryption_keys" # and you have the following file tree: # # ├── encryption_at_rest @@ -317,7 +317,7 @@ scylla_ssl: # without the {{ inventory_hostname }} in it or simply copy the same key for all the node folders. handle_system_keys: false -localhost_system_key_directory: "{{ inventory_dir }}/encryption_at_rest/{{ inventory_hostname }}/system_encryption_keys" +localhost_system_key_directory: "{{ playbook_dir }}/encryption_at_rest/{{ inventory_hostname }}/system_encryption_keys" system_key_directory: "/etc/scylla/system_encryption_keys" # Table keys are used for encrypting SSTables. Depending on your key provider, this key is stored in different locations: @@ -332,7 +332,7 @@ system_key_directory: "/etc/scylla/system_encryption_keys" # Note that by default the {{ inventory_hostname }} is part of the localhost_table_key_directory. This allows you to define # different keys per host. # e.g.: -# If your localhost_table_key_directory is set to "{{ inventory_dir }}/encryption_at_rest/{{ inventory_hostname }}/table_encryption_keys" +# If your localhost_table_key_directory is set to "{{ playbook_dir }}/encryption_at_rest/{{ inventory_hostname }}/table_encryption_keys" # and you have the following file tree: # # ├── encryption_at_rest @@ -352,7 +352,7 @@ system_key_directory: "/etc/scylla/system_encryption_keys" # without the {{ inventory_hostname }} in it or simply copy the same key for all the node folders. handle_table_keys: false -localhost_table_key_directory: "{{ inventory_dir }}/encryption_at_rest/{{ inventory_hostname }}/table_encryption_keys" +localhost_table_key_directory: "{{ playbook_dir }}/encryption_at_rest/{{ inventory_hostname }}/table_encryption_keys" table_key_directory: "/etc/scylla/table_encryption_keys" # Encrypt system resources: diff --git a/ansible-scylla-node/tasks/adjust_keyspace_replication.yml b/ansible-scylla-node/tasks/adjust_keyspace_replication.yml index f926a68a..570ec528 100644 --- a/ansible-scylla-node/tasks/adjust_keyspace_replication.yml +++ b/ansible-scylla-node/tasks/adjust_keyspace_replication.yml @@ -1,14 +1,4 @@ --- -- name: Validate that all nodes are up before adjusting the replication - uri: - url: "http://{{ scylla_api_address }}:{{ scylla_api_port }}/failure_detector/endpoints/" - follow_redirects: none - method: GET - register: _result - until: _result.status == 200 - retries: 10 - delay: 1 - - name: Get datacenter name uri: url: "http://{{ scylla_api_address }}:{{ scylla_api_port }}/snitch/datacenter" @@ -17,16 +7,27 @@ until: _datacenter_out.status == 200 retries: 5 delay: 1 + delegate_to: "{{ item }}" + loop: "{{ groups['scylla'] }}" + run_once: true - name: Prepare per DC replication_factor list set_fact: - dcs_to_rf: "{{ dcs_to_rf | default([]) + [\"'\" + hostvars[item]['_datacenter_out'].json + \"':\" + _keyspace_rf|string] }}" - loop: "{{ groups['scylla'] }}" + dcs_to_rf: "{{ dcs_to_rf | default([]) + [\"'\" + item.json + \"':\" + _keyspace_rf|string] }}" + loop: "{{ _datacenter_out.results }}" run_once: true -- set_fact: - scylla_admin_username: "{{ scylla_admin_default_user }}" - scylla_admin_password: "{{ lookup('ini', scylla_admin_default_user, section='cql_credentials', allow_no_value=true, default=omit, file=inventory_file) }}" +- name: Set scylla_admin_username and scylla_admin_password + block: + - name: Set scylla_admin_username + set_fact: + scylla_admin_username: "{{ scylla_admin_default_user }}" + + - name: Look for cql_credentials in the inventories and read scylla_admin_password + include_tasks: read_admin_password.yml + vars: + iv_file: "{{ item }}" + loop: "{{ ansible_inventory_sources }}" when: scylla_admin_default_user is defined and 'cql_credentials' in groups - fail: diff --git a/ansible-scylla-node/tasks/common.yml b/ansible-scylla-node/tasks/common.yml index 28ccf857..4d75662d 100644 --- a/ansible-scylla-node/tasks/common.yml +++ b/ansible-scylla-node/tasks/common.yml @@ -79,7 +79,7 @@ - name: Create a map from dc to its list of nodes set_fact: dc_to_node_list: "{{ dc_to_node_list | default({}) | combine( {hostvars[item]['dc']: (dc_to_node_list | default({}))[hostvars[item]['dc']] | default([]) + [item]} ) }}" - loop: "{{ groups['scylla'] }}" + loop: "{{ play_hosts }}" run_once: true when: scylla_io_probe_dc_aware|bool @@ -328,13 +328,13 @@ - name: Start seeders serially run_once: true include_tasks: start_one_node.yml - loop: "{{ groups['scylla'] }}" + loop: "{{ play_hosts }}" when: hostvars[item]['broadcast_address'] in scylla_seeds or item in scylla_seeds - name: Start scylla non-seeds nodes serially run_once: true include_tasks: start_one_node.yml - loop: "{{ groups['scylla'] }}" + loop: "{{ play_hosts }}" when: - item not in scylla_seeds - hostvars[item]['broadcast_address'] not in scylla_seeds @@ -345,11 +345,17 @@ host: "{{ scylla_api_address }}" timeout: 300 + - name: Create a list with the broadcast addresses of the nodes in play_hosts + set_fact: + broadcast_address_list: "{{ broadcast_address_list | default([]) + [hostvars[item]['broadcast_address']] }}" + loop: "{{ play_hosts }}" + run_once: true + - name: wait for the cluster to become healthy shell: | - nodetool status|grep -E '^UN|^UJ|^DN'|wc -l + nodetool status|grep -E '^UN|^UJ|^DN'| grep -Ew "{{ broadcast_address_list | join('|') }}" | wc -l register: node_count - until: node_count.stdout|int == ansible_play_batch|length + until: node_count.stdout|int == play_hosts|length retries: "{{ scylla_bootstrap_wait_time_sec|int }}" delay: 1 when: full_inventory|bool @@ -390,6 +396,18 @@ {% if (_scylla_yaml_map.audit is defined and _scylla_yaml_map.audit == 'table') %}True{% else %}False{% endif %} run_once: true +- name: Check if all nodes are up + block: + - name: Check if there are joining or down nodes + shell: | + nodetool status|grep -E '^UJ|^DN' | wc -l + register: node_count + + - name: Set all_nodes_up as a fact + set_fact: + all_nodes_up: "{{ node_count.stdout|int == 0 }}" + run_once: true + - name: Adjust replication for system_auth keyspace include_tasks: adjust_keyspace_replication.yml vars: @@ -397,7 +415,7 @@ _keyspace_replication_strategy: "{{ system_auth_replication_strategy }}" _keyspace_rf: "{{ system_auth_rf }}" when: - - start_scylla_service|bool + - all_nodes_up|bool - adjust_system_auth_replication is defined and adjust_system_auth_replication|bool - _authentication_enabled is defined and _authentication_enabled|bool - system_auth_rf is defined and system_auth_replication_strategy is defined @@ -409,7 +427,7 @@ _keyspace_replication_strategy: "{{ audit_replication_strategy }}" _keyspace_rf: "{{ audit_rf }}" when: - - start_scylla_service|bool + - all_nodes_up|bool - adjust_audit_replication is defined and adjust_audit_replication|bool - _audit_enabled is defined and _audit_enabled|bool - audit_rf is defined and audit_replication_strategy is defined @@ -421,7 +439,7 @@ _keyspace_replication_strategy: "{{ system_distributed_replication_strategy }}" _keyspace_rf: "{{ system_distributed_rf }}" when: - - start_scylla_service|bool + - all_nodes_up|bool - adjust_system_distributed_replication is defined and adjust_system_distributed_replication|bool - system_distributed_rf is defined and system_distributed_replication_strategy is defined @@ -432,7 +450,7 @@ _keyspace_replication_strategy: "{{ system_traces_replication_strategy }}" _keyspace_rf: "{{ system_traces_rf }}" when: - - start_scylla_service|bool + - all_nodes_up|bool - adjust_system_traces_replication is defined and adjust_system_traces_replication|bool - system_traces_rf is defined and system_traces_replication_strategy is defined diff --git a/ansible-scylla-node/tasks/generate_tokens.yml b/ansible-scylla-node/tasks/generate_tokens.yml index 8d417ab3..41bebadb 100644 --- a/ansible-scylla-node/tasks/generate_tokens.yml +++ b/ansible-scylla-node/tasks/generate_tokens.yml @@ -1,4 +1,48 @@ --- +- name: Get relevant addresses for nodes that are not in 'play_hosts' + block: + - name: check if scylla.yaml exists + stat: + path: /etc/scylla/scylla.yaml + register: _scylla_yaml_stat + delegate_to: "{{ item }}" + delegate_facts: true + when: item not in play_hosts + loop: "{{ groups['scylla'] }}" + + - name: Set _scylla_yaml_exists as a fact + set_fact: + _scylla_yaml_exists: "{{ item.stat.exists }}" + when: item.item not in play_hosts + delegate_to: "{{ item.item }}" + delegate_facts: true + loop: "{{ _scylla_yaml_stat.results }}" + + # Let's assume that a node can only be started if it has a scylla yaml + - name: Register scylla.yaml + command: cat /etc/scylla/scylla.yaml + register: _scylla_yaml_out + delegate_to: "{{ item }}" + when: "item not in play_hosts and hostvars[item]['_scylla_yaml_exists']|bool" + loop: "{{ groups['scylla'] }}" + + - name: Set _scylla_yaml_map as a fact + set_fact: + _scylla_yaml_map: "{{ item.stdout | from_yaml }}" + delegate_to: "{{ item.item }}" + delegate_facts: true + when: item.stdout is defined + loop: "{{ _scylla_yaml_out.results }}" + + - name: Set addresses as facts + set_fact: + rpc_address: "{{ hostvars[item]['_scylla_yaml_map'].rpc_address }}" + broadcast_address: "{{ hostvars[item]['_scylla_yaml_map'].broadcast_address }}" + delegate_to: "{{ item }}" + delegate_facts: true + when: item not in play_hosts and hostvars[item]['_scylla_yaml_exists']|bool + loop: "{{ groups['scylla'] }}" + - name: Find the first node which is already bootstrapped, if any wait_for: port: 9042 @@ -7,7 +51,7 @@ register: wait_for_cql_port_output ignore_errors: true delegate_to: "{{ item }}" - loop: "{{ groups['scylla'] }}" + loop: "{{ scylla_seeds }}" when: wait_for_cql_port_output is not defined or wait_for_cql_port_output.failed == True - name: Set the already bootstrapped node as a fact, if any @@ -21,6 +65,7 @@ node_list: "{{ node_list | default([]) + [hostvars[item]['broadcast_address']] }}" rack_list: "{{ rack_list | default([]) + [hostvars[item]['rack']] }}" dc_list: "{{ dc_list | default([]) + [hostvars[item]['dc'] + hostvars[item]['dc_suffix'] | default('')] }}" + when: hostvars[item]['broadcast_address'] is defined loop: "{{ groups['scylla'] }}" - name: Create a temporary tokens file @@ -40,6 +85,7 @@ retries: 5 delay: 1 delegate_to: "{{ bootstrapped_node }}" + when: hostvars[item]['broadcast_address'] is defined loop: "{{ groups['scylla'] }}" - name: Copy tokens to tmp file @@ -47,7 +93,7 @@ path: "{{ tokens_file.path }}" line: "{{ hostvars[item.item]['broadcast_address'] }}={{ item.json | map('int') | join(',') }}" create: yes - when: item.json|length > 0 + when: hostvars[item.item]['broadcast_address'] is defined and item.json|length > 0 delegate_to: localhost loop: "{{ _existing_tokens.results }}" when: bootstrapped_node is defined @@ -77,4 +123,4 @@ when: item[0].split('=')[0] == hostvars[item[1]]['broadcast_address'] with_nested: - "{{ _new_tokens.stdout_lines }}" - - "{{ groups['scylla'] }}" + - "{{ play_hosts }}" diff --git a/ansible-scylla-node/tasks/manager_agents.yml b/ansible-scylla-node/tasks/manager_agents.yml index e66567f7..6250a8fd 100644 --- a/ansible-scylla-node/tasks/manager_agents.yml +++ b/ansible-scylla-node/tasks/manager_agents.yml @@ -2,7 +2,7 @@ - name: check for pre-existing agent-token file stat: - path: "{{ cluster_local_files_path }}/scyllamgr_auth_token.txt" + path: "{{ manager_local_files_path }}/scyllamgr_auth_token.txt" register: token_file_stat run_once: true delegate_to: localhost @@ -20,14 +20,14 @@ copy: content: | {{ scyllamgr_auth_token.stdout }} - dest: "{{ cluster_local_files_path }}/scyllamgr_auth_token.txt" + dest: "{{ manager_local_files_path }}/scyllamgr_auth_token.txt" delegate_to: localhost run_once: true when: token_file_stat.stat.islnk is not defined - name: read the key from auth token file slurp: - src: "{{ cluster_local_files_path }}/scyllamgr_auth_token.txt" + src: "{{ manager_local_files_path }}/scyllamgr_auth_token.txt" register: b64auth_token delegate_to: localhost run_once: true diff --git a/ansible-scylla-node/tasks/read_admin_password.yml b/ansible-scylla-node/tasks/read_admin_password.yml new file mode 100644 index 00000000..cf4f4afd --- /dev/null +++ b/ansible-scylla-node/tasks/read_admin_password.yml @@ -0,0 +1,6 @@ +--- +- name: Set password for scylla_admin + set_fact: + scylla_admin_password: "{{ lookup('ini', scylla_admin_username, section='cql_credentials', allow_no_value=true, default=omit, file=iv_file) }}" + inventory_with_credentials: "{{ iv_file }}" + ignore_errors: true diff --git a/ansible-scylla-node/tasks/ssl.yml b/ansible-scylla-node/tasks/ssl.yml index e87fb5f6..7785494a 100644 --- a/ansible-scylla-node/tasks/ssl.yml +++ b/ansible-scylla-node/tasks/ssl.yml @@ -1,6 +1,6 @@ --- - set_fact: - _localhost_cert_path: "{{ scylla_ssl.localhost_cert_path | default(inventory_dir) }}" + _localhost_cert_path: "{{ scylla_ssl.localhost_cert_path | default(playbook_dir) }}" - name: For every node, check if crt file exists stat: diff --git a/example-playbooks/manage_users/manage_users.yml b/example-playbooks/manage_users/manage_users.yml index b2cb7734..7e966788 100644 --- a/example-playbooks/manage_users/manage_users.yml +++ b/example-playbooks/manage_users/manage_users.yml @@ -63,9 +63,13 @@ when: _authentication_enabled|bool == false run_once: true - - name: Set password for scylla_admin - set_fact: - scylla_admin_password: "{{ lookup('ini', scylla_admin_username, section='cql_credentials', allow_no_value=true, default=omit, file=inventory_file) }}" + - name: Loop through inventories to find scylla_admin credentials + block: + - name: Try to read scylla_admin credentials in the current inventory + include_tasks: ../../ansible-scylla-node/tasks/read_admin_password.yml + vars: + iv_file: "{{ item }}" + loop: "{{ ansible_inventory_sources }}" - name: Create scylla_admin user import_tasks: create_user.yml @@ -85,7 +89,7 @@ admin_username: "{{ scylla_admin_username }}" admin_password: "{{ scylla_admin_password }}" username: "{{ item.key }}" - password: "{{ lookup('ini', item.key, section='cql_credentials', allow_no_value=true, default=omit, file=inventory_file) }}" + password: "{{ lookup('ini', item.key, section='cql_credentials', allow_no_value=true, default=omit, file=inventory_with_credentials) }}" permissions: "{{ item.value.permissions }}" loop: "{{ internal_users | dict2items }}" run_once: true