Support Tiflash (#1119)

* add tiflash config

* Add tiflash

* add deploy tiflash

* update

* fix some error

* fix typo

* optimize labels for tiflash

* optimize format

* reset some config (#1096)

* Merge multiple configuration files and add status check (#1102)

* update tiflash version (#1118)

* Update roles/tiflash/tasks/main.yml

* create status dir for all services

Co-authored-by: Liangliang Gu <[email protected]>

conf/tiflash-learner.yml

@@ -0,0 +1,55 @@
+# TiKV config template
+#  Human-readable big numbers:
+#   File size(based on byte): KB, MB, GB, TB, PB
+#    e.g.: 1_048_576: "1MB"
+#   Time(based on ms): ms, s, m, h
+#    e.g.: 78_000: "1.3m"
+
+readpool:
+  storage:
+
+  coprocessor:
+
+server:
+
+storage:
+
+pd:
+# This section will be overwritten by command line parameters
+
+metric:
+#address: "172.16.30.31:9531"
+#interval: "15s"
+#job: "tikv"
+
+raftstore:
+  raftdb-path: ""
+  sync-log: true
+  max-leader-missing-duration: "22s"
+  abnormal-leader-missing-duration: "21s"
+  peer-stale-state-check-interval: "20s"
+
+coprocessor:
+
+rocksdb:
+  wal-dir: ""
+
+  defaultcf:
+    block-cache-size: "10GB"
+
+  lockcf:
+    block-cache-size: "4GB"
+
+  writecf:
+    block-cache-size: "4GB"
+
+raftdb:
+  defaultcf:
+    block-cache-size: "1GB"
+
+security:
+  ca-path: ""
+  cert-path: ""
+  key-path: ""
+
+import:

conf/tiflash.yml

@@ -0,0 +1,52 @@
+---
+
+global:
+  display_name: "TiFlash"
+  default_profile: "default"
+  mark_cache_size: 5368709120
+  listen_host: "0.0.0.0"
+flash:
+  flash_cluster:
+    refresh_interval: 20
+    update_rule_interval: 5
+    master_ttl: 60
+  proxy:
+
+logger:
+  count: 10
+  size: "1000M"
+  level: "trace"
+application:
+  runAsDaemon: true
+
+raft:
+
+quotas:
+  default:
+    interval:
+      result_rows: 0
+      read_rows: 0
+      execution_time: 0
+      queries: 0
+      errors: 0
+      duration: 3600
+users:
+  readonly:
+    quota: "default"
+    profile: "readonly"
+    password: ""
+    networks:
+      ip: "::/0"
+  default:
+    quota: "default"
+    profile: "default"
+    password: ""
+    networks:
+      ip: "::/0"
+profiles:
+  readonly:
+    readonly: 1
+  default:
+    load_balancing: "random"
+    use_uncompressed_cache: 0
+    max_memory_usage: 10000000000

deploy.yml

@@ -131,6 +131,13 @@
   roles:
     - tikv
 
+- name: deploying TiFlash cluster
+  hosts: tiflash_servers
+  tags:
+    - tiflash
+  roles:
+    - tiflash
+
 - name: deploying pump cluster
   hosts: pump_servers
   tags:

excessive_rolling_update.yml

@@ -381,6 +381,59 @@
 
     - include_tasks: "common_tasks/remove_evict_leader_scheduler.yml"
 
+- name: rolling update TiFlash cluster
+  hosts: tiflash_servers
+  any_errors_fatal: true
+  serial: 1
+  tags:
+    - tiflash
+
+  pre_tasks:
+    - name: stop TiFlash by supervise
+      shell: cd {{ deploy_dir }}/scripts && ./stop_tiflash.sh
+      when: process_supervision == 'supervise'
+
+    - name: stop TiFlash by systemd
+      systemd: name=tiflash-{{ tcp_port }}.service state=stopped
+      become: true
+      when: process_supervision == 'systemd'
+
+    - name: wait until the TiFlash port is down
+      wait_for:
+        host: "{{ ansible_host }}"
+        port: "{{ http_port }}"
+        state: stopped
+        msg: "the TiFlash port {{ http_port }} is not down"
+
+  roles:
+    - tiflash
+
+  post_tasks:
+    - name: start TiFlash by supervise
+      shell: cd {{ deploy_dir }}/scripts && ./start_tiflash.sh
+      when: process_supervision == 'supervise'
+
+    - name: start TiFlash by systemd
+      systemd: name=tiflash-{{ tcp_port }}.service state=started
+      become: true
+      when: process_supervision == 'systemd'
+
+    - name: wait until the TiFlash port is up
+      wait_for:
+        host: "{{ ansible_host }}"
+        port: "{{ http_port }}"
+        state: started
+        msg: "the TiFlash port {{ http_port }} is not up"
+
+    - name: wait until the TiFlash status page is available
+      uri:
+        url: "http://{{ ansible_host }}:{{ http_port }}/?query=select%20version()"
+        return_content: yes
+      register: tiflash_http_result
+      until: tiflash_http_result.status == 200
+      retries: 12
+      delay: 5
+      when: not enable_tls|default(false)
 
 - name: rolling update pump cluster
   hosts: pump_servers

group_vars/tiflash_servers.yml

@@ -0,0 +1,7 @@
+---
+
+tcp_port: 9000
+http_port: 8123
+interserver_http_port: 9009
+flash_service_port: 3930
+flash_proxy_port: 20170

inventory.ini

@@ -49,6 +49,9 @@
 
 [drainer_servers]
 
+## Tiflash Part
+[tiflash_servers]
+
 ## Group variables
 [pd_servers:vars]
 # location_labels = ["zone","rack","host"]

roles/common_dir/tasks/main.yml

@@ -12,7 +12,6 @@
   file: path={{ item }} state=directory mode=0755
   with_items:
     - "{{ status_dir }}"
-  when: deployment_method == 'supervise' or 'tikv_servers' in group_names
 
 - name: create deploy binary directory
   file: path={{ item }} state=directory mode=0755

roles/local/tasks/binary_deployment.yml

@@ -48,6 +48,21 @@
     - has_outbound_network
     - not deploy_without_tidb|default(false)
 
+- name: download TiFlash packages
+  get_url:
+    url: "{{ item.url }}"
+    dest: "{{ downloads_dir }}/{{ item.name }}-{{ item.version }}.tar.gz"
+    checksum: "{{ item.checksum | default(omit) }}"
+    force: yes
+    validate_certs: no
+  register: get_url_result
+  until: "'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg"
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  with_items: "{{ tiflash_packages }}"
+  when:
+    - has_outbound_network
+
 - name: unarchive third party binary
   shell: ls -1 {{ item.name }}-{{ item.version }}.tar.gz | xargs -n1 tar xzf
   args:
@@ -69,6 +84,13 @@
     warn: no
   when: not deploy_without_tidb|default(false)
 
+- name: unarchive tiflash
+  shell: ls -1 {{ item.name }}-{{ item.version }}.tar.gz | xargs tar xzf
+  args:
+    chdir: "{{ downloads_dir }}"
+    warn: no
+  with_items: "{{ tiflash_packages }}"
+
 - name: cp monitoring binary
   shell: >
     cp -v {{ downloads_dir }}/{{ item }}-*/{{ item }} "{{ resources_dir }}/bin/{{ item }}"
@@ -88,3 +110,8 @@
   shell: >
     cp -rfv {{ downloads_dir }}/tispark-sample-data "{{ resources_dir }}/bin/"
   when: not deploy_without_tidb|default(false)
+
+- name: cp tiflash directory
+  shell: >
+    cp -rfv {{ downloads_dir }}/{{ item.name }}-{{ item.version }}-linux-amd64 "{{ resources_dir }}/bin/tiflash"
+  with_items: "{{ tiflash_packages }}"

roles/local/templates/binary_packages.yml.j2

@@ -68,3 +68,8 @@ tispark_packages:
     url: http://download.pingcap.org/tispark-sample-data.tar.gz
     checksum: "sha256:bd0368a9d8663a4a8de89e39cc4cc1d91c718faf36d4bc7e1f8482c34d5bb8db"
 {% endif %}
+
+tiflash_packages:
+  - name: tiflash
+    version: {{ tidb_version }}
+    url: http://download.pingcap.org/tiflash-{{ tidb_version }}-linux-amd64.tar.gz

roles/tiflash/defaults/main.yml

@@ -0,0 +1,20 @@
+---
+
+tiflash_dir: "{{ deploy_dir }}/tiflash"
+tmp_path: "{{ deploy_dir }}/tiflash/data/tmp"
+path: "{{ deploy_dir }}/tiflash/data/db"
+cluster_manager_path: "{{ deploy_dir }}/bin/tiflash/flash_cluster_manager"
+cluster_manager_log: "{{ deploy_dir }}/log/tiflash_cluster_manager.log"
+tiflash_tikv_log: "{{ deploy_dir }}/log/tiflash_tikv.log"
+tiflash_errlog: "{{ deploy_dir }}/log/tiflash_error.log"
+tiflash_server_log: "{{ deploy_dir }}/log/tiflash.log"
+flash_proxy_config: "{{ deploy_dir }}/conf/tiflash-learner.toml"
+data_dir: "{{ deploy_dir }}/tiflash/data/flash"
+tiflash_conf_dir: "{{ deploy_dir }}/conf"
+tiflash_log_dir: "{{ deploy_dir }}/log"
+tiflash_scripts_dir: "{{ deploy_dir }}/scripts"
+tcp_port: 9000
+http_port: 8123
+interserver_http_port: 9009
+flash_service_port: 3930
+flash_proxy_port: 20170

roles/tiflash/meta/main.yml

@@ -0,0 +1,4 @@
+---
+
+dependencies:
+  - role: common_dir

roles/tiflash/tasks/binary_deployment.yml

@@ -0,0 +1,23 @@
+---
+
+- name: deploy tiflash binary
+  copy: src="{{ resources_dir }}/bin/tiflash" dest="{{ deploy_dir }}/bin/" mode=0755 backup=yes
+  register: tiflash_binary
+
+- name: create run script
+  template:
+    src: "{{ item }}_{{ role_name }}_binary.sh.j2"
+    dest: "{{ deploy_dir }}/scripts/{{ item }}_{{ role_name }}.sh"
+    mode: "0755"
+    backup: yes
+  with_items:
+    - run
+  vars:
+    role_status_dir: status/{{ role_name }}
+  register: tiflash_script
+
+- name: backup script file
+  command: mv "{{ tiflash_script.backup_file }}" "{{ backup_dir }}"
+  when: tiflash_script.changed and tiflash_script.backup_file is defined
+
+- include_tasks: "systemd_deployment.yml"

roles/tiflash/tasks/main.yml

@@ -0,0 +1,66 @@
+---
+
+- name: create tiflash directories
+  file: path={{ item }} state=directory mode=0755
+  with_items:
+  - "{{ tiflash_dir }}"
+  - "{{ tmp_path }}"
+  - "{{ path }}"
+  - "{{ data_dir }}"
+  - "{{ tiflash_conf_dir }}"
+  - "{{ tiflash_log_dir }}"
+  - "{{ tiflash_scripts_dir }}"
+
+- name: "load tiflash config: tidb-ansible/conf/tiflash.yml"
+  include_vars: file={{ playbook_dir }}/conf/tiflash.yml name=tiflash_conf_custom
+
+- name: load tiflash default config
+  include_vars: file=tiflash.yml name=tiflash_conf_default
+
+- name: generate tiflash dynamic config
+  set_fact:
+    tiflash_conf_generated:
+      flash:
+        flash_cluster:
+          cluster_manager_path: "{{ cluster_manager_path }}"
+          log: "{{ cluster_manager_log }}"
+
+- name: combine tiflash config
+  set_fact:
+    tiflash_conf: "{{ tiflash_conf_custom | with_default_dicts(tiflash_conf_generated, tiflash_conf_default) | update_default_dicts }}"
+
+- debug: var=tiflash_conf
+
+- name: create tiflash config file
+  template: src=tiflash.toml.j2 dest={{ deploy_dir }}/conf/tiflash.toml mode=0644 backup=yes
+  register: tiflash_conf_st
+
+- name: backup tiflash conf file
+  command: mv "{{ tiflash_conf_st.backup_file }}" "{{ backup_dir }}"
+  when: tiflash_conf_st.changed and tiflash_conf_st.backup_file is defined
+
+- name: "load tiflash learner config: tidb-ansible/conf/tiflash-learner.yml"
+  include_vars: file={{ playbook_dir }}/conf/tiflash-learner.yml name=tiflash_learner_conf_custom
+
+- name: load tiflash learner default config
+  include_vars: file=tiflash-learner.yml name=tiflash_learner_conf_default
+
+- name: generate tiflash learner config
+  set_fact:
+    tiflash_learner_conf: "{{ tiflash_learner_conf_custom | with_default_dicts(tiflash_learner_conf_default) | update_default_dicts }}"
+
+- debug: var=tiflash_learner_conf
+
+- name: create tiflash learner config file
+  template: src=tiflash_learner.toml.j2 dest={{ deploy_dir }}/conf/tiflash-learner.toml mode=0644 backup=yes
+  register: tiflash_learner_conf_st
+
+- name: backup tiflash conf file
+  command: mv "{{ tiflash_learner_conf_st.backup_file }}" "{{ backup_dir }}"
+  when: tiflash_learner_conf_st.changed and tiflash_learner_conf_st.backup_file is defined
+
+- include_tasks: "binary_deployment.yml"
+
+- name: prepare firewalld white list
+  set_fact:
+    firewalld_ports: "{{ [tcp_port ~ '/tcp', http_port ~ '/tcp', interserver_http_port ~ '/tcp', flash_service_port ~ '/tcp', flash_proxy_port ~ '/tcp'] + firewalld_ports }}"

roles/tiflash/tasks/supervise_deployment.yml

@@ -0,0 +1,8 @@
+---
+
+- name: deploy supervise
+  include_role:
+    name: supervise
+  vars:
+    this_role_name: tiflash
+    service_name: tiflash-{{ tcp_port }}