excessive_rolling_update.yml

---
# Copyright 2016 PingCAP, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# See the License for the specific language governing permissions and
# limitations under the License.

# The rolling update playbook of TiDB

- name: check config locally
  hosts: localhost
  any_errors_fatal: true
  tags:
    - always
  roles:
    - check_config_static

- name: check system environment
  hosts: monitored_servers
  any_errors_fatal: true
  tags:
    - always
  roles:
    - check_system_dynamic

- name: gather all facts, and check dest
  hosts: all
  any_errors_fatal: true
  tags:
    - always
  roles:
    - check_config_dynamic

- name: Pre-check for rolling update
  hosts: tidb_servers
  any_errors_fatal: true
  tags:
    - always
  tasks:
    - shell: "{{ deploy_dir }}/bin/tidb-server -V"
      register: current_version

    - name: Check whether can perform rolling update
      fail:
        msg: "Rolling update from {{ current_version.stdout_lines[0].replace(' ','').split(':')[1] }} to {{ tidb_version }} is forbidden"
      when:
        - current_version.stdout_lines[0].replace(' ','').split(':')[1] < "v2.0.1"
        - tidb_version >= "v2.1.0" or tidb_version == "latest"

- name: Pre-check PD configuration
  hosts: pd_servers[0]
  tags:
    - pd
  roles:
    - check_config_pd

- name: Pre-check TiKV configuration
  hosts: tikv_servers[0]
  tags:
    - tikv
  roles:
    - check_config_tikv

- name: Pre-check TiDB configuration
  hosts: tidb_servers[0]
  tags:
    - tidb
  roles:
    - check_config_tidb

- hosts: pd_servers[0]
  any_errors_fatal: true
  serial: 1
  tags:
    - pd
  tasks:
    - name: Check pd cluster status
      uri:
        url: "http://{{ ansible_host }}:{{ pd_client_port }}/pd/health"
        method: GET
        return_content: yes
        status_code: 200
      register: pd_status
      when: not enable_tls|default(false)

    - name: Check pd cluster status when enable_tls
      uri:
        url: "https://{{ ansible_host }}:{{ pd_client_port }}/pd/health"
        validate_certs: no
        client_cert: "{{ pd_cert_dir }}/pd-server-{{ ansible_host }}.pem"
        client_key: "{{ pd_cert_dir }}/pd-server-{{ ansible_host }}-key.pem"
        method: GET
        return_content: yes
        status_code: 200
      register: pd_status_tls
      when: enable_tls|default(false)

    - name: Failed when one node of pd is unhealthy
      fail:
        msg: "Some pd nodes are unhealthy"
      when:
        - not enable_tls|default(false)
        - "'false' in pd_status.content"

    - name: Failed when one node of pd is unhealthy when enable_tls
      fail:
        msg: "Some pd nodes are unhealthy"
      when:
        - enable_tls|default(false)
        - "'false' in pd_status_tls.content"

- hosts: pd_servers
  any_errors_fatal: true
  serial: 1
  tags:
    - pd
  tasks:
    - set_fact:
        pd_addr: "{{ ansible_host }}:{{ pd_client_port }}"

    - include_tasks: "common_tasks/get_pd_leader.yml"
      when: not enable_tls|default(false)

    - include_tasks: "common_tasks/get_pd_leader_tls.yml"
      when: enable_tls|default(false)

    - set_fact:
        pd_leader_name: "{{ pd_leader_info.json.name }}"

    - include_tasks: "common_tasks/get_pd_name.yml"
      when: not enable_tls|default(false)

    - include_tasks: "common_tasks/get_pd_name_tls.yml"
      when: enable_tls|default(false)

    - name: Set pd follower list
      add_host:
        name: "{{ inventory_hostname }}"
        ansible_host: "{{ ansible_host }}"
        ansible_ssh_host: "{{ ansible_ssh_host }}"
        groups: pd_servers_followers
        deploy_dir: "{{ deploy_dir }}"
        pd_client_port: "{{ pd_client_port }}"
        pd_peer_port: "{{ pd_peer_port }}"
        pd_data_dir: "{{ pd_data_dir }}"
        pd_log_dir: "{{ pd_log_dir }}"
        pd_cert_dir: "{{ pd_cert_dir }}"
      when: pd_leader_name != pd_name

    - name: Set pd leader list
      add_host:
        name: "{{ inventory_hostname }}"
        ansible_host: "{{ ansible_host }}"
        ansible_ssh_host: "{{ ansible_ssh_host }}"
        groups: pd_servers_leader
        deploy_dir: "{{ deploy_dir }}"
        pd_client_port: "{{ pd_client_port }}"
        pd_peer_port: "{{ pd_peer_port }}"
        pd_data_dir: "{{ pd_data_dir }}"
        pd_log_dir: "{{ pd_log_dir }}"
        pd_cert_dir: "{{ pd_cert_dir }}"
      when: pd_leader_name == pd_name

- name: rolling update PD cluster
  hosts: pd_servers_followers, pd_servers_leader
  any_errors_fatal: true
  serial: 1
  tags:
    - pd

  pre_tasks:
    - set_fact:
        pd_addr: "{{ ansible_host }}:{{ pd_client_port }}"

    - include_tasks: "common_tasks/get_pd_name.yml"
      when: not enable_tls|default(false)

    - include_tasks: "common_tasks/get_pd_name_tls.yml"
      when: enable_tls|default(false)

    - name: display PD name
      debug:
        var: pd_name

    - name: display PD address
      debug:
        var: pd_addr

    - include_tasks: "common_tasks/get_pd_leader.yml"
      when: not enable_tls|default(false)

    - include_tasks: "common_tasks/get_pd_leader_tls.yml"
      when: enable_tls|default(false)

    - include_tasks: "common_tasks/transfer_pd_leader.yml"

    - name: stop PD by supervise
      shell: cd {{ deploy_dir }}/scripts && ./stop_pd.sh
      when: process_supervision == 'supervise'

    - name: stop PD by systemd
      systemd: name=pd.service state=stopped
      become: true
      when: process_supervision == 'systemd'

    - name: wait until the PD port is down
      wait_for:
        host: "{{ ansible_host }}"
        port: "{{ pd_client_port }}"
        state: stopped
        msg: "the PD port {{ pd_client_port }} is not down"

  roles:
    - pd

  post_tasks:
    - name: start PD by supervise
      shell: cd {{ deploy_dir }}/scripts && ./start_pd.sh
      when: process_supervision == 'supervise'

    - name: start PD by systemd
      systemd: name=pd-{{ pd_client_port }}.service state=started
      become: true
      when: process_supervision == 'systemd'

    - name: wait until the PD port is up
      wait_for:
        host: "{{ ansible_host }}"
        port: "{{ pd_client_port }}"
        state: started
        msg: "the PD port {{ pd_client_port }} is not up"

    - name: wait until the PD health page is available
      uri:
        url: "http://{{ ansible_host }}:{{ pd_client_port }}/health"
        return_content: yes
      register: pd_http_result
      until: pd_http_result.status == 200 and 'true' in pd_http_result.content
      retries: 12
      delay: 5
      when: not enable_tls|default(false)

    - name: wait until the PD health page is available when enable_tls
      uri:
        url: "https://{{ ansible_host }}:{{ pd_client_port }}/health"
        validate_certs: no
        client_cert: "{{ pd_cert_dir }}/pd-server-{{ ansible_host }}.pem"
        client_key: "{{ pd_cert_dir }}/pd-server-{{ ansible_host }}-key.pem"
        return_content: yes
      register: pd_https_result
      until: pd_https_result.status == 200 and 'true' in pd_https_result.content
      retries: 12
      delay: 5
      when: enable_tls|default(false)

    - name: wait until the PD cluster is available
      uri:
        url: "http://{{ ansible_host }}:{{ pd_client_port }}/pd/health"
        return_content: yes
      register: pd_cluster_status
      until: pd_cluster_status.status == 200 and 'false' not in pd_cluster_status.content
      retries: 12
      delay: 5
      when: not enable_tls|default(false)

    - name: wait until the PD cluster is available when enable_tls
      uri:
        url: "https://{{ ansible_host }}:{{ pd_client_port }}/pd/health"
        validate_certs: no
        client_cert: "{{ pd_cert_dir }}/pd-server-{{ ansible_host }}.pem"
        client_key: "{{ pd_cert_dir }}/pd-server-{{ ansible_host }}-key.pem"
        return_content: yes
      register: pd_cluster_status
      until: pd_cluster_status.status == 200 and 'false' not in pd_cluster_status.content
      retries: 12
      delay: 5
      when: enable_tls|default(false)


- name: rolling update TiKV cluster
  hosts: tikv_servers
  any_errors_fatal: true
  serial: 1
  tags:
    - tikv

  pre_tasks:
    - include_tasks: "common_tasks/get_pd_tikv_addr.yml"

    - include_tasks: "common_tasks/get_store_id.yml"
      when: not enable_tls|default(false)

    - include_tasks: "common_tasks/get_store_id_tls.yml"
      when: enable_tls|default(false)

    - include_tasks: "common_tasks/add_evict_leader_scheduler.yml"

    - name: stop TiKV by supervise
      shell: cd {{ deploy_dir }}/scripts && ./stop_tikv.sh
      when: process_supervision == 'supervise'

    - name: stop TiKV by systemd
      systemd: name=tikv-{{ tikv_port }}.service state=stopped
      become: true
      when: process_supervision == 'systemd'

    - name: wait until the TiKV port is down
      wait_for:
        host: "{{ ansible_host }}"
        port: "{{ tikv_port }}"
        state: stopped
        msg: "the TiKV port {{ tikv_port }} is not down"

    - command: cat {{ deploy_dir }}/status/tikv.pid
      register: old_tikv_pid
      ignore_errors: yes
      changed_when: false

    - name: display old tikv pid
      debug:
        msg: "tikv binary or docker pid: {{ old_tikv_pid.stdout }}"

  roles:
    - tikv

  post_tasks:
    - name: start TiKV by supervise
      shell: cd {{ deploy_dir }}/scripts && ./start_tikv.sh
      when: process_supervision == 'supervise'

    - name: start TiKV by systemd
      systemd: name=tikv-{{ tikv_port }}.service state=started
      become: true
      when: process_supervision == 'systemd'

    - name: wait until the TiKV port is up
      wait_for:
        host: "{{ ansible_host }}"
        port: "{{ tikv_port }}"
        state: started
        msg: "the TiKV port {{ tikv_port }} is not up"

    - name: wait until the TiKV status page is available
      uri:
        url: "http://{{ ansible_host }}:{{ tikv_status_port }}/status"
        return_content: yes
      register: tikv_http_result
      until: tikv_http_result.status == 200
      retries: 12
      delay: 5
      when: not enable_tls|default(false)

    - name: wait until the TiKV status page is available when enable_tls
      uri:
        url: "https://{{ ansible_host }}:{{ tikv_status_port }}/status"
        validate_certs: no
        client_cert: "{{ tikv_cert_dir }}/tikv-server-{{ ansible_host }}.pem"
        client_key: "{{ tikv_cert_dir }}/tikv-server-{{ ansible_host }}-key.pem"
        return_content: yes
      register: tikv_https_result
      until: tikv_https_result.status == 200
      retries: 10
      delay: 5
      when: enable_tls|default(false)

    - command: cat {{ deploy_dir }}/status/tikv.pid
      register: new_tikv_pid
      ignore_errors: yes
      changed_when: false

    - name: display new tikv pid
      debug:
        msg: "tikv binary or docker pid: {{ new_tikv_pid.stdout }}"

    - include_tasks: "common_tasks/remove_evict_leader_scheduler.yml"


- name: rolling update pump cluster
  hosts: pump_servers
  any_errors_fatal: true
  serial: 1
  tags:
    - pump

  pre_tasks:
    - name: stop pump by supervise
      shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh
      with_items:
        - pump
      when:
        - enable_binlog|default(false)
        - process_supervision == 'supervise'

    - name: stop pump by systemd
      systemd: name=pump-{{ pump_port }}.service state=stopped
      become: true
      when:
        - enable_binlog|default(false)
        - process_supervision == 'systemd'

    - name: wait until the pump port is down
      wait_for:
        host: "{{ ansible_host }}"
        port: "{{ pump_port }}"
        state: stopped
        msg: "the pump port {{ pump_port }} is not down"
      when:
        - enable_binlog|default(false)

  roles:
    - { role: pump, when: enable_binlog|default(false) }

  post_tasks:
    - name: start pump by supervise
      shell: cd {{ deploy_dir }}/scripts && ./start_{{ item }}.sh
      when:
        - enable_binlog|default(false)
        - process_supervision == 'supervise'
      with_items:
        - pump

    - name: start pump by systemd
      systemd: name=pump-{{ pump_port }}.service state=started
      become: true
      when:
        - enable_binlog|default(false)
        - process_supervision == 'systemd'

    - name: wait until the pump port is up
      wait_for:
        host: "{{ ansible_host }}"
        port: "{{ pump_port }}"
        state: started
        msg: "the pump port {{ pump_port }} is not up"
      when:
        - enable_binlog|default(false)


- name: rolling update TiDB cluster
  hosts: tidb_servers
  any_errors_fatal: true
  serial: 1
  tags:
    - tidb

  pre_tasks:
    - name: stop TiDB by supervise
      shell: cd {{ deploy_dir }}/scripts && ./stop_tidb.sh
      when: process_supervision == 'supervise'

    - name: stop TiDB by systemd
      systemd: name=tidb-{{ tidb_port }}.service state=stopped
      become: true
      when: process_supervision == 'systemd'

    - name: wait until the TiDB port is down
      wait_for:
        host: "{{ ansible_host }}"
        port: "{{ tidb_port }}"
        state: stopped
        msg: "the TiDB port {{ tidb_port }} is not down"

  roles:
    - { role: tidb }

  post_tasks:

    - name: start TiDB by supervise
      shell: cd {{ deploy_dir }}/scripts && ./start_tidb.sh
      when: process_supervision == 'supervise'

    - name: start TiDB by systemd
      systemd: name=tidb-{{ tidb_port }}.service state=started
      become: true
      when: process_supervision == 'systemd'

    - name: wait until the TiDB port is up
      wait_for:
        host: "{{ ansible_host }}"
        port: "{{ tidb_port }}"
        state: started
        msg: "the TiDB port {{ tidb_port }} is not up"

    - name: wait until the TiDB status page is available
      uri:
        url: "http://{{ ansible_host }}:{{ tidb_status_port }}/status"
        return_content: yes
      register: tidb_http_result
      until: tidb_http_result.status == 200 and 'TiDB' in tidb_http_result.content
      retries: 12
      delay: 5
      when: not enable_tls|default(false)

    - name: wait until the TiDB status page is available when enable_tls
      uri:
        url: "https://{{ ansible_host }}:{{ tidb_status_port }}/status"
        validate_certs: no
        client_cert: "{{ tidb_cert_dir }}/tidb-server-{{ ansible_host }}.pem"
        client_key: "{{ tidb_cert_dir }}/tidb-server-{{ ansible_host }}-key.pem"
        return_content: yes
      register: tidb_https_result
      until: tidb_https_result.status == 200 and 'TiDB' in tidb_https_result.content
      retries: 10
      delay: 5
      when: enable_tls|default(false)