Merge pull request #16 from krzyzakp/master

Fixing travis/runtime errors. Adding few more config to management thanks to @krzyzakp
AlexeySetevoi · Aug 22, 2018 · b2a0d2c · b2a0d2c
2 parents 2701eb7 + 637eaee
commit b2a0d2c
Show file tree

Hide file tree

Showing 14 changed files with 118 additions and 38 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -28,18 +28,6 @@ matrix:
       dist: xenial
       sudo: true
       env: ANSIBLE_VERSION='latest'
-    - os: linux
-      dist: precise
-      sudo: true
-      env: ANSIBLE_VERSION='ansible>=2.4,<2.5'
-    - os: linux
-      dist: precise
-      sudo: true
-      env: ANSIBLE_VERSION='ansible>=2.3,<2.4'
-    - os: linux
-      dist: precise
-      sudo: true
-      env: ANSIBLE_VERSION='latest'
 
 # Use the new container infrastructure
 sudo: true
@@ -94,4 +82,4 @@ script:
 
 
 notifications:
-  webhooks: https://galaxy.ansible.com/api/v1/notifications/
+  webhooks: https://galaxy.ansible.com/api/v1/notifications/
diff --git a/defaults/main.yml b/defaults/main.yml
@@ -79,3 +79,4 @@ clickhouse_config:
 
 clickhouse_dicts: []
 
+clickhouse_path_user_files: /var/lib/clickhouse/user_files/
diff --git a/meta/main.yml b/meta/main.yml
@@ -11,7 +11,10 @@ galaxy_info:
       versions:
         - xenial
         - trusty
-        - precise
+    - name: Debian
+      versions:
+        - jessie
+        - stretch
 
   galaxy_tags:
     - clickhouse

diff --git a/tasks/config_db.yml b/tasks/config_db.yml
@@ -1,6 +1,6 @@
 ---
 - name: Gather list of existing databases
-  command: "clickhouse-client -h localhost -q 'show databases'"
+  command: "clickhouse-client -h 127.0.0.1 -q 'show databases'"
   changed_when: False
   register: existing_databases
   tags: [config_db]
@@ -12,7 +12,7 @@
   tags: [config_db]
 
 - name: Config | Create database config
-  command: "clickhouse-client -h localhost -q 'CREATE DATABASE IF NOT EXISTS `{{ item.name }}`'"
+  command: "clickhouse-client -h 127.0.0.1 -q 'CREATE DATABASE IF NOT EXISTS `{{ item.name }}`'"
   with_items: "{{ clickhouse_dbs }}"
   when: (item.state is undefined or item.state == 'present') and item.name not in existing_databases.stdout_lines
   tags: [config_db]
diff --git a/tasks/config_sys.yml b/tasks/config_sys.yml
@@ -14,23 +14,52 @@
 
 - name: Config | Generate system config
   template:
-   src: config.j2
-   dest: "{{ clickhouse_path_configdir }}/config.xml"
-   owner: clickhouse
-   group: clickhouse
+    src: config.j2
+    dest: "{{ clickhouse_path_configdir }}/config.xml"
+    owner: clickhouse
+    group: clickhouse
   notify: restart-ch
   become: true
 
 - name: Config | Generate users config
   template:
-   src: users.j2
-   dest: "{{ clickhouse_path_configdir }}/users.xml"
-   owner: clickhouse
-   group: clickhouse
+    src: users.j2
+    dest: "{{ clickhouse_path_configdir }}/users.xml"
+    owner: clickhouse
+    group: clickhouse
   notify: restart-ch
   become: true
 
-- meta: flush_handlers
+- name: Config | Generate remote_servers config
+  template:
+    src: remote_servers.j2
+    dest: "{{ clickhouse_path_configdir }}/conf.d/clickhouse_remote_servers.xml"
+    owner: clickhouse
+    group: clickhouse
+  notify: restart-ch
+  become: true
+  when: clickhouse_shards is defined 
 
+- name: Config | Generate macros config
+  template:
+    src: macros.j2
+    dest: "{{ clickhouse_path_configdir }}/conf.d/macros.xml"
+    owner: clickhouse
+    group: clickhouse
+    mode: 0644
+  notify: restart-ch
+  become: true
+  when: clickhouse_macros is defined 
 
+- name: Config | Generate zookeeper servers config
+  template:
+    src: zookeeper-servers.j2
+    dest: "{{ clickhouse_path_configdir }}/conf.d/zookeeper-servers.xml"
+    owner: clickhouse
+    group: clickhouse
+    mode: 0644
+  notify: restart-ch
+  become: true
+  when: clickhouse_zookeeper_nodes is defined 
 
+- meta: flush_handlers
diff --git a/tasks/precheck.yml b/tasks/precheck.yml
@@ -15,3 +15,4 @@
   ignore_errors: yes
   when: ansible_pkg_mgr == 'apt' and clickhouse_setup == 'package'
   changed_when: False
+  check_mode: no
diff --git a/templates/config.j2 b/templates/config.j2
@@ -1,4 +1,5 @@
 <?xml version="1.0"?>
+<!-- {{ ansible_managed }} -->
 <yandex>
     <logger>
         <level>{{ clickhouse_logger.level }}</level>
@@ -58,12 +59,14 @@
     <!--
     <interserver_http_host>example.yandex.ru</interserver_http_host>
     -->
-
+{% if clickhouse_interserver_http_host is defined %}
+    <interserver_http_host>{{ clickhouse_interserver_http_host }}</interserver_http_host>
+{% endif %}
     <!-- Listen specified host. use :: (wildcard IPv6 address), if you want to accept connections both with IPv4 and IPv6 from everywhere. -->
     <!-- <listen_host>::</listen_host> -->
-    {% for host in clickhouse_listen_host %}
+{% for host in clickhouse_listen_host %}
     <listen_host>{{ host }}</listen_host>
-    {% endfor %}
+{% endfor %}
 
     <max_connections>{{ clickhouse_config.max_connections }}</max_connections>
     <keep_alive_timeout>{{ clickhouse_config.keep_alive_timeout }}</keep_alive_timeout>
@@ -94,7 +97,8 @@
 
     <!-- Path to temporary data for processing hard queries. -->
     <tmp_path>{{ clickhouse_path_tmp }}</tmp_path>
-
+
+    <user_files_path>{{ clickhouse_path_user_files }}</user_files_path>
     <!-- Path to configuration file with users, access rights, profiles of settings, quotas. -->
     <users_config>users.xml</users_config>
 
@@ -201,7 +205,7 @@
         -->
         <database>system</database>
         <table>query_log</table>
-
+        <partition_by>toYYYYMM(event_date)</partition_by>
         <!-- Interval of flushing data. -->
         <flush_interval_milliseconds>7500</flush_interval_milliseconds>
     </query_log>
@@ -307,4 +311,7 @@
             </retention>
         </default>
     </graphite_rollup_example>
+
+  <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
+
 </yandex>
diff --git a/templates/macros.j2 b/templates/macros.j2
@@ -0,0 +1,9 @@
+<?xml version="1.0"?>
+<!-- {{ ansible_managed }} -->
+<yandex>
+  <macros>
+{% for key, value in clickhouse_macros.items() %}
+    <{{ key }}>{{ value }}</{{ key }}>
+{% endfor %}
+  </macros>
+</yandex>
diff --git a/templates/remote_servers.j2 b/templates/remote_servers.j2
@@ -0,0 +1,19 @@
+<?xml version="1.0"?>
+<!-- {{ ansible_managed }} -->
+<yandex>
+  <remote_servers>
+{% for shard_name, replicas in clickhouse_shards.iteritems() %}
+    <{{ shard_name }}>
+      <shard>
+        <internal_replication>true</internal_replication>
+{% for replica in replicas %}
+        <replica>
+          <host>{{ replica['host'] }}</host>
+          <port>{{ replica['port'] | default(9000) }}</port>
+        </replica>
+{% endfor %}
+      </shard>
+    </{{ shard_name }}>
+{% endfor %}
+  </remote_servers>
+</yandex>
diff --git a/templates/zookeeper-servers.j2 b/templates/zookeeper-servers.j2
@@ -0,0 +1,13 @@
+<?xml version="1.0"?>
+<!-- {{ ansible_managed }} -->
+<yandex>
+  <zookeeper>
+{% for server in clickhouse_zookeeper_nodes %}
+    <node>
+{% for key, value in server.items() %}
+      <{{ key }}>{{ value }}</{{ key }}>
+{% endfor %}
+    </node>
+{% endfor %}
+  </zookeeper>
+</yandex>
diff --git a/vars/Ubuntu_precise.yml → vars/Debian_jessie.yml b/vars/Ubuntu_precise.yml → vars/Debian_jessie.yml
@@ -1,14 +1,12 @@
 clickhouse_supported: yes
 clickhouse_repo: "deb http://repo.yandex.ru/clickhouse/deb/stable/ main/"
-clickhouse_repo_old: "deb http://repo.yandex.ru/clickhouse/xenial stable main"
 clickhouse_repo_keyserver: keyserver.ubuntu.com
 clickhouse_repo_key: E0C56BD4
 clickhouse_package:
   - clickhouse-client
-  - clickhouse-server-base
-  - clickhouse-server-common
+  - clickhouse-common-static
+  - clickhouse-server
 clickhouse_path_configdir: "/etc/clickhouse-server"
 clickhouse_path_logdir: "/var/log/clickhouse-server"
 clickhouse_path_data: "/var/lib/clickhouse/"
 clickhouse_path_tmp: "/var/lib/clickhouse/tmp/"
-
diff --git a/vars/Debian_stretch.yml b/vars/Debian_stretch.yml
@@ -0,0 +1,12 @@
+clickhouse_supported: yes
+clickhouse_repo: "deb http://repo.yandex.ru/clickhouse/deb/stable/ main/"
+clickhouse_repo_keyserver: keyserver.ubuntu.com
+clickhouse_repo_key: E0C56BD4
+clickhouse_package:
+  - clickhouse-client
+  - clickhouse-common-static
+  - clickhouse-server
+clickhouse_path_configdir: "/etc/clickhouse-server"
+clickhouse_path_logdir: "/var/log/clickhouse-server"
+clickhouse_path_data: "/var/lib/clickhouse/"
+clickhouse_path_tmp: "/var/lib/clickhouse/tmp/"
diff --git a/vars/Ubuntu_trusty.yml b/vars/Ubuntu_trusty.yml
@@ -5,8 +5,8 @@ clickhouse_repo_keyserver: keyserver.ubuntu.com
 clickhouse_repo_key: E0C56BD4
 clickhouse_package:
   - clickhouse-client
-  - clickhouse-server-base
-  - clickhouse-server-common
+  - clickhouse-common-static
+  - clickhouse-server
 clickhouse_path_configdir: "/etc/clickhouse-server"
 clickhouse_path_logdir: "/var/log/clickhouse-server"
 clickhouse_path_data: "/var/lib/clickhouse/"

diff --git a/vars/Ubuntu_xenial.yml b/vars/Ubuntu_xenial.yml
@@ -5,8 +5,8 @@ clickhouse_repo_keyserver: keyserver.ubuntu.com
 clickhouse_repo_key: E0C56BD4
 clickhouse_package:
   - clickhouse-client
-  - clickhouse-server-base
-  - clickhouse-server-common
+  - clickhouse-common-static
+  - clickhouse-server
 clickhouse_path_configdir: "/etc/clickhouse-server"
 clickhouse_path_logdir: "/var/log/clickhouse-server"
 clickhouse_path_data: "/var/lib/clickhouse/"
Original file line number	Diff line number	Diff line change
Expand Up		@@ -79,3 +79,4 @@ clickhouse_config:

		clickhouse_dicts: []

		clickhouse_path_user_files: /var/lib/clickhouse/user_files/