Merge pull request #1444 from ingvagabund/refactor-kubernetes-roles

Automatic merge from submit-queue

Refactor kubernetes roles

Continuing effort [1] of cleaning roles for kubernetes, master and node roles.

TODO:
- [x] clean ``kubernetes`` role
- [x] move ``download_bins.yml`` under ``node`` and ``master`` and enable installation of downloaded binaries for Fedora, CentOS and other OSes.
- [x] extend the installation options with installation of distribution rpm (not just the latest)
- [x] clean and refactor ``node`` role
- [x] clean and refactor ``master`` role
- [x] introduce ``deploy-master.yml`` and ``deploy-node.yml`` playbooks, include them in the top level playbook
- [x] tag relevant tasks and introduce new scripts
- [x] move CPU and Memory accounting under node role (inside ``kubelet-configuration.yml``)

Some items of the TODO (new playbooks, new scripts) can be resolved only once the #967 is merged.

[1] #1298

ansible/README.md

@@ -66,6 +66,41 @@ You can just setup certain parts instead of doing it all.
 
 `$ ./deploy-cluster.sh --tags=nodes`
 
+### Component sources
+
+Each component can be installed from various sources. For instance:
+
+* distribution packages
+* github release
+* kubernetes built from local source codes
+
+By default, every component (etcd, docker, kubernetes, etc.) is installed via distribution package manager.
+Currently, the following component types are supported:
+
+* `etcd_source_type`: for `etcd` role
+* `flannel_source_type`: for `flannel` role
+* `kube_source_type`: for `master` and `node` roles
+* `source_type`: for other roles (and components)
+
+To see a full list of available types, see corresponding role's default variables.
+
+#### Kubernetes source type
+
+Available types (see `kube_source_type` under `roles/kubernetes/defaults/main.yml`):
+
+* `packageManager`
+* `localBuild`
+* `github-release`
+* `distribution-rpm`
+
+In case of a package manager, the `kube-apiserver` binary is shipped with `cap_net_bind_service=ep` capability set.
+The capability allows the apiserver to listen on `443` port.
+In a case of `localBuild` and `github-release`, the capability is not set.
+In order for apiserver to listen on a secure port, change the port (see `kube_master_api_port` under `roles/kubernetes/defaults/main.yml`). For instance to listen on `6443`.
+
+In order to apply the `distribution-rpm` type, location of an rpm must be specified.
+See `kube_rpm_url_base` and `kube_rpm_url_sufix` variables under `roles/kubernetes/defaults/main.yml`.
+
 ### Network Service
 
 By changing the `networking` variable in the `inventory/group_vars/all.yml` file, you can choose the network-service to use.  The default is flannel.
@@ -74,6 +109,6 @@ By changing the `networking` variable in the `inventory/group_vars/all.yml` file
 
 ### Troubleshooting
 
-* When updating flannel to version ``0.5.5-7`` or higher on Fedora, ``/etc/sysconfig/flannel`` configuration file (if changed) must be updated to reflect renamed systemd environment variables.
+* When updating flannel to version `0.5.5-7` or higher on Fedora, the `/etc/sysconfig/flannel` configuration file (if changed) must be updated to reflect renamed systemd environment variables.
 
 [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/contrib/ansible/README.md?pixel)]()

ansible/playbooks/deploy-cluster.yml

@@ -47,12 +47,7 @@
     - { role: contiv, contiv_role: netmaster, when: networking == 'contiv' }
 
 # install kube master services
-- hosts: masters
-  become: yes
-  roles:
-    - master
-  tags:
-    - masters
+- include: deploy-master.yml
 
 # launch addons, like dns
 - hosts: masters
@@ -64,12 +59,7 @@
     - dns
 
 # install kubernetes on the nodes
-- hosts: nodes
-  become: yes
-  roles:
-    - node
-  tags:
-    - nodes
+- include: deploy-node.yml
 
 # provision opencontrail once the services are operational
 - hosts:
@@ -87,10 +77,3 @@
   roles:
     - { role: contiv, contiv_role: netplugin, when: networking == 'contiv' }
 
-# install runtime dependencies
-- hosts: nodes
-  become: yes
-  roles:
-  - epilogue
-  tags:
-  - epilogue

ansible/playbooks/deploy-etcd.yml

@@ -1,6 +1,5 @@
 ---
-
-- name: Setup etcd
+- name: Deploy etcd
   hosts: etcd
   sudo: yes
   roles:

ansible/playbooks/deploy-master.yml

@@ -0,0 +1,9 @@
+---
+- name: Deploy master
+  hosts: masters
+  become: yes
+  roles:
+    - master
+  tags:
+    - masters
+

ansible/playbooks/deploy-node.yml

@@ -0,0 +1,9 @@
+---
+- name: Deploy node
+  hosts: nodes
+  become: yes
+  roles:
+    - node
+  tags:
+    - nodes
+

ansible/roles/common/tasks/main.yml

@@ -48,14 +48,3 @@
 
 - include: fedora-install.yml
   when: not is_atomic and ansible_distribution == "Fedora"
-
-# enable cpu and memory accounting for systemd services by default
-- name: Create systemd dropin directory
-  file: path=/etc/systemd/system/system.conf.d state=directory mode=0755
-  when: source_type == "localBuild"
-
-- name: Enable cpu and memory accounting for systemd services
-  copy: src=kubernetes-accounting.conf dest="/etc/systemd/system/system.conf.d/kubernetes-accounting.conf"
-  when: source_type == "localBuild"
-  notify:
-    - reload systemd

ansible/roles/kubernetes/defaults/main.yml

@@ -1,6 +1,10 @@
 # Version of Kubernetes binaries
 kube_version: 1.2.4
 
+# Set source of kubernetes binaries
+# Available: packageManager, localBuild, github-release, distribution-rpm
+kube_source_type: packageManager
+
 # The port that the Kubernetes apiserver component listens on.
 kube_master_api_port: 443
 
@@ -20,14 +24,9 @@ kube_config_dir: /etc/kubernetes
 # The URL to download Kubernetes binaries from.
 kube_download_url_base: https://storage.googleapis.com/kubernetes-release/release/v{{ kube_version }}/bin/linux/amd64
 
-# List of Kubernetes binaries to download and install.
-kube_binaries:
-  - kube-apiserver
-  - kube-controller-manager
-  - kube-proxy
-  - kube-scheduler
-  - kubectl
-  - kubelet
+# The URL do download distribution rpms shipping kubernetes binaries from
+kube_rpm_url_base: https://kojipkgs.fedoraproject.org//packages/kubernetes/1.2.0/0.27.git4a3f9c5.fc25/x86_64
+kube_rpm_url_sufix: 1.2.0-0.27.git4a3f9c5.fc25.x86_64.rpm
 
 # Directory to store downloaded Kubernetes releases
 kube_releases_directory: /opt/kubernetes

ansible/roles/kubernetes/tasks/configure.yml

@@ -0,0 +1,9 @@
+---
+- name: Create kubernetes config directory
+  file: path={{ kube_config_dir }} state=directory
+
+- name: Write the global config file
+  template: src=config.j2 dest={{ kube_config_dir }}/config
+  notify:
+    - restart daemons
+

ansible/roles/kubernetes/tasks/main.yml

@@ -1,40 +1,17 @@
 ---
-- include: fedora.yml
-  when: ansible_distribution == "Fedora"
+- name: Fedora | Remove docker window manager on F20
+  yum: pkg=docker state=absent
+  when: ansible_distribution == "Fedora" and ansible_distribution_major_version == "20"
+  tags: install
 
-- name: Update {{ kube_script_dir }} if this is atomic
-  set_fact:
-    kube_script_dir: "/usr/local/libexec/kubernetes"
-  when: is_atomic and kube_script_dir == "/usr/libexec/kubernetes"
+- name: Override kubernetes facts
+  include: override_facts.yml
 
-- name: Initialize the python_bin fact
-  set_fact:
-    python_bin: "python"
-
-- name: Update facts if this is CoreOS or Ubuntu 14.04
-  set_fact:
-    bin_dir: "/opt/bin"
-    kube_script_dir: "/opt/bin/kubernetes"
-    python_bin: "/opt/bin/python"
-  when: is_coreos or ansible_distribution == 'Ubuntu' and ansible_distribution_major_version|int < 15
-
-- name: Create kubernetes config directory
-  file: path={{ kube_config_dir }} state=directory
-
-- name: Create kubernetes script directory
-  file: path={{ kube_script_dir }} state=directory
-
-- name: Create bin dir
-  file: path={{ bin_dir }} state=directory
-
-- name: write the global config file
-  template: src=config.j2 dest={{ kube_config_dir }}/config
-  notify:
-    - restart daemons
-
-- include: download_bins.yml
-  when: is_coreos or ansible_distribution == 'Ubuntu' and ansible_distribution_major_version|int < 15
+- name: Configure kubernetes
+  include: configure.yml
+  tags: configure
 
 - include: secrets.yml
   tags:
     secrets
+  tags: configure

ansible/roles/kubernetes/tasks/override_facts.yml

@@ -0,0 +1,17 @@
+---
+- name: Update {{ kube_script_dir }} if this is atomic
+  set_fact:
+    kube_script_dir: "/usr/local/libexec/kubernetes"
+  when: is_atomic and kube_script_dir == "/usr/libexec/kubernetes"
+
+- name: Initialize the python_bin fact
+  set_fact:
+    python_bin: "python"
+
+- name: Update facts if this is CoreOS or Ubuntu 14.04
+  set_fact:
+    bin_dir: "/opt/bin"
+    kube_script_dir: "/opt/bin/kubernetes"
+    python_bin: "/opt/bin/python"
+  when: is_coreos or ansible_distribution == 'Ubuntu' and ansible_distribution_major_version|int < 15
+

ansible/roles/kubernetes/tasks/secrets.yml

@@ -1,4 +1,7 @@
 ---
+- name: Create kubernetes script directory
+  file: path={{ kube_script_dir }} state=directory
+
 - name: Create system kube-cert groups
   group: name={{ kube_cert_group }} state=present system=yes
 

ansible/roles/master/defaults/main.yml

@@ -6,3 +6,20 @@ admission_controllers: NamespaceLifecycle,NamespaceExists,LimitRanger,ServiceAcc
 
 kube_apiserver_bind_address: "0.0.0.0"
 kube_apiserver_insecure_bind_address: "127.0.0.1"
+
+services_enabled: true
+apiserver_modified: false
+controller_manager_modified: false
+scheduler_modified: false
+kubelet_modified: false
+
+# List of Kubernetes binaries to download and install.
+kube_master_binaries:
+  - kube-apiserver
+  - kube-controller-manager
+  - kube-scheduler
+  - kubectl
+
+kube_master_rpms:
+  - kubernetes-client
+  - kubernetes-master

ansible/roles/master/tasks/apiserver-configure.yml

@@ -0,0 +1,16 @@
+---
+- name: Write the config file for the api server
+  template: src=apiserver.j2 dest={{ kube_config_dir }}/apiserver
+
+- name: Ensure that a token auth file exists (addons may populate it)
+  file: path={{ kube_token_dir }}/known_tokens.csv state=touch
+  changed_when: false
+
+- name: Add cap_net_bind_service to kube-apiserver
+  capabilities: path=/usr/bin/kube-apiserver capability=cap_net_bind_service=ep state=present
+  when: kube_source_type != "github-release" and not is_atomic and not is_coreos and not ansible_distribution == 'Ubuntu' and not ansible_distribution_major_version|int < 15
+
+- name: Setting the apiserver_modified fact to true
+  set_fact:
+    apiserver_modified: true
+

ansible/roles/master/tasks/apiserver-restart.yml

@@ -0,0 +1,4 @@
+---
+- name: Restart apiserver
+  service: name=kube-apiserver state=restarted
+

ansible/roles/master/tasks/apiserver-start.yml

@@ -0,0 +1,8 @@
+---
+- name: Enable apiserver
+  service: name=kube-apiserver enabled=yes
+  when: services_enabled == true
+
+- name: Start apiserver
+  service: name=kube-apiserver state=started
+  register: apiserver_started

ansible/roles/master/tasks/collect-tokens.yml

@@ -0,0 +1,17 @@
+---
+- name: Get the master token values
+  slurp:
+    src: "{{ kube_token_dir }}/{{ item }}-{{ inventory_hostname }}.token"
+  with_items:
+    - "system:controller_manager"
+    - "system:scheduler"
+    - "system:kubectl"
+  register: tokens
+  delegate_to: "{{ groups['masters'][0] }}"
+
+- name: Set token facts
+  set_fact:
+    controller_manager_token: "{{ tokens.results[0].content|b64decode }}"
+    scheduler_token: "{{ tokens.results[1].content|b64decode }}"
+    kubectl_token: "{{ tokens.results[2].content|b64decode }}"
+

ansible/roles/master/tasks/controller-manager-configure.yml

@@ -0,0 +1,11 @@
+---
+- name: Write the config file for the controller-manager
+  template: src=controller-manager.j2 dest={{ kube_config_dir }}/controller-manager
+
+- name: Write the kubecfg (auth) file for controller-manager
+  template: src=controller-manager.kubeconfig.j2 dest={{ kube_config_dir }}/controller-manager.kubeconfig
+
+- name: Setting the controller_manager_modified fact to true
+  set_fact:
+    controller_manager_modified: true
+

ansible/roles/master/tasks/controller-manager-restart.yml

@@ -0,0 +1,4 @@
+---
+- name: Restart controller-manager
+  service: name=kube-controller-manager state=restarted
+

ansible/roles/master/tasks/controller-manager-start.yml

@@ -0,0 +1,8 @@
+---
+- name: Enable controller-manager
+  service: name=kube-controller-manager enabled=yes
+  when: services_enabled == true
+
+- name: Start controller-manager
+  service: name=kube-controller-manager state=started
+  register: controller_manager_started

ansible/roles/master/tasks/coreos.yml

@@ -1,8 +1,4 @@
 ---
-- name: CoreOS | Force source_type to github
-  set_fact:
-    source_type: "github-release"
-
 - name: CoreOS | Get Systemd Unit Files from Kubernetes repository
   get_url:
     url=https://raw.githubusercontent.com/kubernetes/contrib/master/init/systemd/{{ item }}.service