Containerd change (#2919)

* Switch to containerd (#2769)

* Add containerd

* Switch to containerd if upgrade

* Add changelog entry

* Spec test update

* Rebase CHANGELOG-2.0.md

* Adjust filebeat.yml after docker removal

* Adjust K8s cert renewal + fix in spec test

* Change filebeat custom chart values and spec test, fix backup k8s

* Doc update

* Add extra condition for docker removal

* Adjust filebeat.yml after docker removal

* Changes after review

* Doc update

* Hardcode containerd version, update components.md

* Changes after review

* Adjust filebeat.yml after docker removal

* Changes after review

* Add crictl config file, add containerd version in docker role

* Add ansible_managed comment

* Enable containerd service, Add fix for single machine

* Add spec tests for containerd service

* Add annotation to k8s/masterN, fix for upgrade

* k8s/master0 -> k8s/masterN

* Add retry to kubectl command

ansible/playbooks/kubernetes_master.yml

@@ -8,7 +8,7 @@
   become_method: sudo
   pre_tasks:
     - import_role:
-        name: docker
+        name: containerd
         tasks_from: main
     - import_role:
         name: kubernetes_common

ansible/playbooks/kubernetes_node.yml

@@ -20,7 +20,7 @@
   become_method: sudo
   pre_tasks:
     - import_role:
-        name: docker
+        name: containerd
         tasks_from: main
     - import_role:
         name: kubernetes_common

ansible/playbooks/roles/backup/tasks/kubernetes.yml

@@ -49,11 +49,13 @@
 
     - name: Save etcd snapshot
       shell: |
-        docker run \
-        -v "{{ backup_temp_dir.path }}/:/backup/" \
-        --network host \
+        ctr --namespace k8s.io \
+        run \
+        --mount type=bind,src={{ backup_temp_dir.path }}/,dst=/backup/,options=rbind:rw \
+        --net-host \
         --env ETCDCTL_API=3 \
         --rm "{{ etcd_image_name.stdout | trim }}" \
+        etcd \
         etcdctl \
         --endpoints https://127.0.0.1:2379 \
         --cacert /backup/pki/etcd/ca.crt \

ansible/playbooks/roles/containerd/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+containerd_version: "1.4.12"

ansible/playbooks/roles/containerd/files/containerd.conf

@@ -0,0 +1,3 @@
+# Ansible managed
+overlay
+br_netfilter

ansible/playbooks/roles/containerd/files/crictl.yaml

@@ -0,0 +1,2 @@
+# Ansible managed
+runtime-endpoint: unix:///run/containerd/containerd.sock

ansible/playbooks/roles/containerd/handlers/main.yml

@@ -0,0 +1,10 @@
+- name: Restart containerd
+  systemd:
+    name: containerd
+    state: restarted
+    enabled: true
+
+- name: Restart kubelet
+  systemd:
+    name: kubelet
+    state: restarted

ansible/playbooks/roles/containerd/tasks/configure-containerd.yml

@@ -0,0 +1,18 @@
+---
+- name: Create containerd dir
+  file:
+    path: /etc/containerd
+    state: directory
+    owner: root
+    group: root
+    mode: u=rw,go=r
+
+- name: Provide containerd config
+  template:
+    src: config.toml.j2
+    dest: /etc/containerd/config.toml
+    mode: u=rw,go=
+    owner: root
+    group: root
+  notify:
+    - Restart containerd

ansible/playbooks/roles/containerd/tasks/configure-kubelet-env.yml

@@ -0,0 +1,39 @@
+# To be replaced by filter plugin (https://github.com/epiphany-platform/epiphany/issues/2943)
+- name: Reconfigure kubelet args
+  notify:
+    - Restart containerd
+    - Restart kubelet
+  block:
+  - name: Get kubeadm-flags.env file content
+    slurp:
+      src: /var/lib/kubelet/kubeadm-flags.env
+    register: kubelet_kubeadm_args
+
+  - name: Set kubelet_kubeadmn_args_content
+    set_fact:
+      kubelet_kubeadmn_args_content: "{{ kubelet_kubeadm_args.content | b64decode }}"
+
+  - name: Modify container-runtime
+    replace:
+      path: /var/lib/kubelet/kubeadm-flags.env
+      regexp: '{{ item.regexp }}'
+      replace: '{{ item.replace }}'
+      backup: true
+    loop:
+      - { regexp: '^(KUBELET_KUBEADM_ARGS=\")(.*)(--container-runtime=[a-zA-Z0-9_]+)(.*)', replace: '\1\2--container-runtime=remote\4' }
+      - { regexp: '^(KUBELET_KUBEADM_ARGS=\")(.*)(--container-runtime-endpoint=.*\.sock)(.*)', replace: '\1\2--container-runtime-endpoint=/run/containerd/containerd.sock\4' }
+    when: kubelet_kubeadmn_args_content.find('--container-runtime') != -1
+
+  - name: Append container-runtime to kubelet config
+    replace:
+      path: /var/lib/kubelet/kubeadm-flags.env
+      regexp: '^(KUBELET_KUBEADM_ARGS=\")(.*)(\")$'
+      replace: '\1\2 --container-runtime=remote"'
+    when: kubelet_kubeadmn_args_content.find('--container-runtime') == -1
+
+  - name: Append container-runtime-endpoint to kubelet config
+    replace:
+      path: /var/lib/kubelet/kubeadm-flags.env
+      regexp: '^(KUBELET_KUBEADM_ARGS=\")(.*)(\")$'
+      replace: '\1\2 --container-runtime-endpoint=/run/containerd/containerd.sock"'
+    when: kubelet_kubeadmn_args_content.find('--container-runtime-endpoint') == -1

ansible/playbooks/roles/containerd/tasks/configure-prerequisites.yml

@@ -0,0 +1,16 @@
+---
+- name: Provide containerd.conf file
+  copy:
+    src: containerd.conf
+    dest: /etc/modules-load.d/containerd.conf
+    owner: root
+    group: root
+    mode: u=rw,go=
+
+- name: Load modules
+  modprobe:
+    name: "{{ item }}"
+    state: present
+  loop:
+    - overlay
+    - br_netfilter

ansible/playbooks/roles/containerd/tasks/main.yml

@@ -0,0 +1,53 @@
+---
+- name: Get information on installed packages
+  package_facts:
+  when: ansible_facts.packages is undefined
+
+- name: Remove Docker
+  block:
+  - name: Stop Kubelet before Docker removal
+    systemd:
+      name: kubelet
+      state: stopped
+    notify:
+    - Restart kubelet
+
+  - name: Remove Docker
+    include_role:
+      name: docker
+      tasks_from: remove-docker.yml
+  when:
+    - is_upgrade_run
+    - inventory_hostname not in groups.image_registry
+    - ansible_facts.packages['docker-ce'] is defined
+
+- name: Install containerd package
+  package:
+    name: "{{ _packages[ansible_os_family] }}"
+    state: present
+  vars:
+    _packages:
+      Debian:
+        - containerd.io={{ containerd_version }}-*
+      RedHat:
+        - containerd.io-{{ containerd_version }}
+  module_defaults:
+    yum: { lock_timeout: "{{ yum_lock_timeout }}" }
+
+- name: Configure prerequisites
+  include_tasks: configure-prerequisites.yml
+
+- name: Configure containerd
+  include_tasks: configure-containerd.yml
+
+- name: Configure kubelet-env
+  include_tasks: configure-kubelet-env.yml
+  when: is_upgrade_run
+
+- name: Provide crictl.yaml file
+  copy:
+    src: crictl.yaml
+    dest: /etc/crictl.yaml
+    owner: root
+    group: root
+    mode: u=rw,go=r

ansible/playbooks/roles/containerd/templates/config.toml.j2

@@ -0,0 +1,131 @@
+# {{ ansible_managed }}
+version = 2
+root = "/var/lib/containerd"
+state = "/run/containerd"
+plugin_dir = ""
+disabled_plugins = []
+required_plugins = []
+oom_score = 0
+
+[grpc]
+  address = "/run/containerd/containerd.sock"
+  tcp_address = ""
+  tcp_tls_cert = ""
+  tcp_tls_key = ""
+  uid = 0
+  gid = 0
+  max_recv_message_size = 16777216
+  max_send_message_size = 16777216
+
+[ttrpc]
+  address = ""
+  uid = 0
+  gid = 0
+
+[debug]
+  address = ""
+  uid = 0
+  gid = 0
+  level = ""
+
+[metrics]
+  address = ""
+  grpc_histogram = false
+
+[cgroup]
+  path = ""
+
+[timeouts]
+  "io.containerd.timeout.shim.cleanup" = "5s"
+  "io.containerd.timeout.shim.load" = "5s"
+  "io.containerd.timeout.shim.shutdown" = "3s"
+  "io.containerd.timeout.task.state" = "2s"
+
+[plugins]
+  [plugins."io.containerd.gc.v1.scheduler"]
+    pause_threshold = 0.02
+    deletion_threshold = 0
+    mutation_threshold = 100
+    schedule_delay = "0s"
+    startup_delay = "100ms"
+  [plugins."io.containerd.grpc.v1.cri"]
+    disable_tcp_service = true
+    stream_server_address = "127.0.0.1"
+    stream_server_port = "0"
+    stream_idle_timeout = "4h0m0s"
+    enable_selinux = false
+    selinux_category_range = 1024
+    sandbox_image = "{{ image_registry_address }}/k8s.gcr.io/pause:3.5"
+    stats_collect_period = 10
+    systemd_cgroup = false
+    enable_tls_streaming = false
+    max_container_log_line_size = 16384
+    disable_cgroup = false
+    disable_apparmor = false
+    restrict_oom_score_adj = false
+    max_concurrent_downloads = 3
+    disable_proc_mount = false
+    unset_seccomp_profile = ""
+    tolerate_missing_hugetlb_controller = true
+    disable_hugetlb_controller = true
+    ignore_image_defined_volumes = false
+    [plugins."io.containerd.grpc.v1.cri".containerd]
+      snapshotter = "overlayfs"
+      default_runtime_name = "runc"
+      no_pivot = false
+      disable_snapshot_annotations = true
+      discard_unpacked_layers = false
+      [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
+        runtime_type = ""
+        runtime_engine = ""
+        runtime_root = ""
+        privileged_without_host_devices = false
+       base_runtime_spec = ""
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          runtime_type = "io.containerd.runc.v2"
+          runtime_engine = ""
+          runtime_root = ""
+          privileged_without_host_devices = false
+          base_runtime_spec = ""
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+            SystemdCgroup = true
+    [plugins."io.containerd.grpc.v1.cri".cni]
+      bin_dir = "/opt/cni/bin"
+      conf_dir = "/etc/cni/net.d"
+      max_conf_num = 1
+      conf_template = ""
+    [plugins."io.containerd.grpc.v1.cri".registry]
+      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
+        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ image_registry_address }}"]
+          endpoint = ["http://{{ image_registry_address }}"]
+        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
+          endpoint = ["https://registry-1.docker.io"]
+    [plugins."io.containerd.grpc.v1.cri".image_decryption]
+      key_model = ""
+    [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
+      tls_cert_file = ""
+      tls_key_file = ""
+  [plugins."io.containerd.internal.v1.opt"]
+    path = "/opt/containerd"
+  [plugins."io.containerd.internal.v1.restart"]
+    interval = "10s"
+  [plugins."io.containerd.metadata.v1.bolt"]
+    content_sharing_policy = "shared"
+  [plugins."io.containerd.monitor.v1.cgroups"]
+    no_prometheus = false
+  [plugins."io.containerd.runtime.v1.linux"]
+    shim = "containerd-shim"
+    runtime = "runc"
+    runtime_root = ""
+    no_shim = false
+    shim_debug = false
+  [plugins."io.containerd.runtime.v2.task"]
+    platforms = ["linux/amd64"]
+  [plugins."io.containerd.service.v1.diff-service"]
+    default = ["walking"]
+  [plugins."io.containerd.snapshotter.v1.devmapper"]
+    root_path = ""
+    pool_name = ""
+    base_image_size = ""
+    async_remove = false

ansible/playbooks/roles/docker/tasks/main.yml

@@ -1,5 +1,9 @@
 ---
-# Docker (used by master & worker as dependency)
+# Docker (used by image_registry)
+- name: Include containerd defauts
+  include_vars:
+    file: roles/containerd/defaults/main.yml
+    name: containerd_defaults
 
 - name: Install Docker packages
   package:
@@ -8,12 +12,12 @@
   vars:
     _packages:
       Debian:
-        - containerd.io
+        - containerd.io={{ containerd_defaults.containerd_version }}-*
         - docker-ce-cli={{ docker_version.Debian }}
         - docker-ce-rootless-extras={{ docker_version.Debian }}
         - docker-ce={{ docker_version.Debian }}
       RedHat:
-        - containerd.io
+        - containerd.io-{{ containerd_defaults.containerd_version }}
         - docker-ce-cli-{{ docker_version.RedHat }}
         - docker-ce-rootless-extras-{{ docker_version.RedHat }}
         - docker-ce-{{ docker_version.RedHat }}

ansible/playbooks/roles/docker/tasks/remove-docker.yml

@@ -0,0 +1,33 @@
+# Included in containerd role
+---
+- name: Populate service facts
+  service_facts:
+  when: ansible_facts.services is undefined
+
+- name: Stop Docker daemon
+  systemd:
+    name: docker
+    state: stopped
+    enabled: no
+  when:
+  - "'docker.service' in ansible_facts.services"
+  - ansible_facts.services['docker.service']['status'] != 'not-found'
+
+- name: Remove Docker packages
+  package:
+    name: "{{ _packages }}"
+    state: absent
+  vars:
+    _packages:
+      - docker-ce-cli
+      - docker-ce-rootless-extras
+      - docker-ce
+
+- name: Remove Docker sockets leftovers
+  file:
+    path: "{{ _paths }}"
+    state: absent
+  vars:
+    _paths:
+      - /var/run/docker.sock
+      - /var/run/dockershim.sock