Skip to content

Commit

Permalink
Make client indentity by AME cert (#11946) (#12908)
Browse files Browse the repository at this point in the history
  • Loading branch information
mssonicbld authored Dec 2, 2022
1 parent 0234596 commit 7152e84
Show file tree
Hide file tree
Showing 5 changed files with 148 additions and 27 deletions.
2 changes: 1 addition & 1 deletion files/build_templates/sonic_debian_extension.j2
Original file line number Diff line number Diff line change
Expand Up @@ -446,7 +446,7 @@ sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install watchd
{% if include_kubernetes == "y" %}
# Point to kubelet to /etc/resolv.conf
#
echo 'KUBELET_EXTRA_ARGS="--resolv-conf=/etc/resolv.conf"' | sudo tee -a $FILESYSTEM_ROOT/etc/default/kubelet
echo 'KUBELET_EXTRA_ARGS="--resolv-conf=/etc/resolv.conf --cgroup-driver=cgroupfs --node-ip=::"' | sudo tee -a $FILESYSTEM_ROOT/etc/default/kubelet

# Copy Flannel conf file into sonic-templates
#
Expand Down
2 changes: 2 additions & 0 deletions src/sonic-ctrmgrd/ctrmgr/ctrmgrd.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,10 +104,12 @@ def log_debug(m):


def log_error(m):
msg = "{}: {}".format(inspect.stack()[1][3], m)
syslog.syslog(syslog.LOG_ERR, msg)


def log_info(m):
msg = "{}: {}".format(inspect.stack()[1][3], m)
syslog.syslog(syslog.LOG_INFO, msg)


Expand Down
82 changes: 76 additions & 6 deletions src/sonic-ctrmgrd/ctrmgr/kube_commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,17 +13,24 @@
import syslog
import tempfile
import urllib.request
import base64
from urllib.parse import urlparse

import yaml
import requests
from sonic_py_common import device_info
from jinja2 import Template
from swsscommon import swsscommon

KUBE_ADMIN_CONF = "/etc/sonic/kube_admin.conf"
KUBELET_YAML = "/var/lib/kubelet/config.yaml"
SERVER_ADMIN_URL = "https://{}/admin.conf"
LOCK_FILE = "/var/lock/kube_join.lock"
FLANNEL_CONF_FILE = "/usr/share/sonic/templates/kube_cni.10-flannel.conflist"
CNI_DIR = "/etc/cni/net.d"
K8S_CA_URL = "https://{}:{}/api/v1/namespaces/default/configmaps/kube-root-ca.crt"
AME_CRT = "/etc/sonic/credentials/restapiserver.crt"
AME_KEY = "/etc/sonic/credentials/restapiserver.key"

def log_debug(m):
msg = "{}: {}".format(inspect.stack()[1][3], m)
Expand Down Expand Up @@ -77,8 +84,7 @@ def _run_command(cmd, timeout=5):

def kube_read_labels():
""" Read current labels on node and return as dict. """
KUBECTL_GET_CMD = "kubectl --kubeconfig {} get nodes --show-labels |\
grep {} | tr -s ' ' | cut -f6 -d' '"
KUBECTL_GET_CMD = "kubectl --kubeconfig {} get nodes {} --show-labels |tr -s ' ' | cut -f6 -d' '"

labels = {}
ret, out, _ = _run_command(KUBECTL_GET_CMD.format(
Expand Down Expand Up @@ -211,6 +217,68 @@ def _download_file(server, port, insecure):
log_debug("{} downloaded".format(KUBE_ADMIN_CONF))


def _gen_cli_kubeconf(server, port, insecure):
"""generate identity which can help authenticate and
authorization to k8s cluster
"""
client_kubeconfig_template = """
apiVersion: v1
clusters:
- cluster:
certificate-authority-data: {{ k8s_ca }}
server: https://{{ vip }}:{{ port }}
name: kubernetes
contexts:
- context:
cluster: kubernetes
user: user
name: user@kubernetes
current-context: user@kubernetes
kind: Config
preferences: {}
users:
- name: user
user:
client-certificate-data: {{ ame_crt }}
client-key-data: {{ ame_key }}
"""
if insecure:
r = requests.get(K8S_CA_URL.format(server, port), cert=(AME_CRT, AME_KEY), verify=False)
else:
r = requests.get(K8S_CA_URL.format(server, port), cert=(AME_CRT, AME_KEY))
if not r.ok:
raise requests.RequestException("Something wrong with AME cert or something wrong about sonic role in k8s cluster")
k8s_ca = r.json()["data"]["ca.crt"]
k8s_ca_b64 = base64.b64encode(k8s_ca.encode("utf-8")).decode("utf-8")
ame_crt_raw = open(AME_CRT, "rb")
ame_crt_b64 = base64.b64encode(ame_crt_raw.read()).decode("utf-8")
ame_key_raw = open(AME_KEY, "rb")
ame_key_b64 = base64.b64encode(ame_key_raw.read()).decode("utf-8")
client_kubeconfig_template_j2 = Template(client_kubeconfig_template)
client_kubeconfig = client_kubeconfig_template_j2.render(
k8s_ca=k8s_ca_b64, vip=server, port=port, ame_crt=ame_crt_b64, ame_key=ame_key_b64)
(h, fname) = tempfile.mkstemp(suffix="_kube_join")
os.write(h, client_kubeconfig.encode("utf-8"))
os.close(h)
log_debug("Downloaded = {}".format(fname))

shutil.copyfile(fname, KUBE_ADMIN_CONF)

log_debug("{} downloaded".format(KUBE_ADMIN_CONF))


def _get_local_ipv6():
try:
config_db = swsscommon.DBConnector("CONFIG_DB", 0)
mgmt_ip_data = swsscommon.Table(config_db, 'MGMT_INTERFACE')
for key in mgmt_ip_data.getKeys():
if key.find(":") >= 0:
return key.split("|")[1].split("/")[0]
raise IOError("IPV6 not find from MGMT_INTERFACE table")
except Exception as e:
raise IOError(str(e))


def _troubleshoot_tips():
""" log troubleshoot tips which could be handy,
when in trouble with join
Expand Down Expand Up @@ -264,12 +332,14 @@ def _do_reset(pending_join = False):


def _do_join(server, port, insecure):
KUBEADM_JOIN_CMD = "kubeadm join --discovery-file {} --node-name {}"
KUBEADM_JOIN_CMD = "kubeadm join --discovery-file {} --node-name {} --apiserver-advertise-address {}"
err = ""
out = ""
ret = 0
try:
_download_file(server, port, insecure)
local_ipv6 = _get_local_ipv6()
#_download_file(server, port, insecure)
_gen_cli_kubeconf(server, port, insecure)
_do_reset(True)
_run_command("modprobe br_netfilter")
# Copy flannel.conf
Expand All @@ -279,11 +349,11 @@ def _do_join(server, port, insecure):

if ret == 0:
(ret, out, err) = _run_command(KUBEADM_JOIN_CMD.format(
KUBE_ADMIN_CONF, get_device_name()), timeout=60)
KUBE_ADMIN_CONF, get_device_name(), local_ipv6), timeout=60)
log_debug("ret = {}".format(ret))

except IOError as e:
err = "Download failed: {}".format(str(e))
err = "Join failed: {}".format(str(e))
ret = -1
out = ""

Expand Down
23 changes: 22 additions & 1 deletion src/sonic-ctrmgrd/tests/common_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
CONFIG_DB_NO = 4
STATE_DB_NO = 6
FEATURE_TABLE = "FEATURE"
MGMT_INTERFACE_TABLE = "MGMT_INTERFACE"
KUBE_LABEL_TABLE = "KUBE_LABELS"
KUBE_LABEL_SET_KEY = "SET"

Expand Down Expand Up @@ -41,6 +42,7 @@
IMAGE_TAG = "image_tag"
FAIL_LOCK = "fail_lock"
DO_JOIN = "do_join"
REQ = "req"

# subproc key words

Expand Down Expand Up @@ -643,8 +645,27 @@ def mock_subproc_side_effect(cmd, shell=False, stdout=None, stderr=None):
return mock_proc(cmd, index)


def set_kube_mock(mock_subproc):
class mock_reqget:
def __init__(self):
self.ok = True

def json(self):
return current_test_data.get(REQ, "")


def mock_reqget_side_effect(url, cert, verify=True):
return mock_reqget()


def set_kube_mock(mock_subproc, mock_table=None, mock_conn=None, mock_reqget=None):
mock_subproc.side_effect = mock_subproc_side_effect
if mock_table != None:
mock_table.side_effect = table_side_effect
if mock_conn != None:
mock_conn.side_effect = conn_side_effect
if mock_reqget != None:
mock_reqget.side_effect = mock_reqget_side_effect


def create_remote_ctr_config_json():
str_conf = '\
Expand Down
66 changes: 47 additions & 19 deletions src/sonic-ctrmgrd/tests/kube_commands_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
KUBE_ADMIN_CONF = "/tmp/kube_admin.conf"
FLANNEL_CONF_FILE = "/tmp/flannel.conf"
CNI_DIR = "/tmp/cni/net.d"
AME_CRT = "/tmp/restapiserver.crt"
AME_KEY = "/tmp/restapiserver.key"

# kube_commands test cases
# NOTE: Ensure state-db entry is complete in PRE as we need to
Expand All @@ -25,8 +27,7 @@
common_test.DESCR: "read labels",
common_test.RETVAL: 0,
common_test.PROC_CMD: ["\
kubectl --kubeconfig {} get nodes --show-labels |\
grep none | tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)],
kubectl --kubeconfig {} get nodes none --show-labels |tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)],
common_test.PROC_OUT: ["foo=bar,hello=world"],
common_test.POST: {
"foo": "bar",
Expand All @@ -39,8 +40,7 @@
common_test.TRIGGER_THROW: True,
common_test.RETVAL: -1,
common_test.PROC_CMD: ["\
kubectl --kubeconfig {} get nodes --show-labels |\
grep none | tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)],
kubectl --kubeconfig {} get nodes none --show-labels |tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)],
common_test.POST: {
},
common_test.PROC_KILLED: 1
Expand All @@ -49,8 +49,7 @@
common_test.DESCR: "read labels fail",
common_test.RETVAL: -1,
common_test.PROC_CMD: ["\
kubectl --kubeconfig {} get nodes --show-labels |\
grep none | tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)],
kubectl --kubeconfig {} get nodes none --show-labels |tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)],
common_test.PROC_OUT: [""],
common_test.PROC_ERR: ["command failed"],
common_test.POST: {
Expand All @@ -65,8 +64,7 @@
common_test.RETVAL: 0,
common_test.ARGS: { "foo": "bar", "hello": "World!", "test": "ok" },
common_test.PROC_CMD: [
"kubectl --kubeconfig {} get nodes --show-labels |\
grep none | tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF),
"kubectl --kubeconfig {} get nodes none --show-labels |tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF),
"kubectl --kubeconfig {} label --overwrite nodes none hello-".format(
KUBE_ADMIN_CONF),
"kubectl --kubeconfig {} label --overwrite nodes none hello=World! test=ok".format(
Expand All @@ -79,8 +77,7 @@
common_test.RETVAL: 0,
common_test.ARGS: { "foo": "bar", "hello": "world" },
common_test.PROC_CMD: [
"kubectl --kubeconfig {} get nodes --show-labels |\
grep none | tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)
"kubectl --kubeconfig {} get nodes none --show-labels |tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)
],
common_test.PROC_OUT: ["foo=bar,hello=world"]
},
Expand All @@ -90,8 +87,7 @@
common_test.ARGS: { "any": "thing" },
common_test.RETVAL: -1,
common_test.PROC_CMD: [
"kubectl --kubeconfig {} get nodes --show-labels |\
grep none | tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)
"kubectl --kubeconfig {} get nodes none --show-labels |tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)
],
common_test.PROC_ERR: ["read failed"]
}
Expand All @@ -114,10 +110,22 @@
"mkdir -p {}".format(CNI_DIR),
"cp {} {}".format(FLANNEL_CONF_FILE, CNI_DIR),
"systemctl start kubelet",
"kubeadm join --discovery-file {} --node-name none".format(
"kubeadm join --discovery-file {} --node-name none --apiserver-advertise-address FC00:2::32".format(
KUBE_ADMIN_CONF)
],
common_test.PROC_RUN: [True, True]
common_test.PROC_RUN: [True, True],
common_test.PRE: {
common_test.CONFIG_DB_NO: {
common_test.MGMT_INTERFACE_TABLE: {
"eth0|FC00:2::32/64": {
"gwaddr": "fc00:2::1"
}
}
}
},
common_test.REQ: {
"data": {"ca.crt": "test"}
}
},
1: {
common_test.DESCR: "Regular secure join",
Expand All @@ -135,10 +143,22 @@
"mkdir -p {}".format(CNI_DIR),
"cp {} {}".format(FLANNEL_CONF_FILE, CNI_DIR),
"systemctl start kubelet",
"kubeadm join --discovery-file {} --node-name none".format(
"kubeadm join --discovery-file {} --node-name none --apiserver-advertise-address FC00:2::32".format(
KUBE_ADMIN_CONF)
],
common_test.PROC_RUN: [True, True]
common_test.PROC_RUN: [True, True],
common_test.PRE: {
common_test.CONFIG_DB_NO: {
common_test.MGMT_INTERFACE_TABLE: {
"eth0|FC00:2::32/64": {
"gwaddr": "fc00:2::1"
}
}
}
},
common_test.REQ: {
"data": {"ca.crt": "test"}
}
},
2: {
common_test.DESCR: "Skip join as already connected",
Expand Down Expand Up @@ -228,11 +248,17 @@ def init(self):
s.close()
with open(FLANNEL_CONF_FILE, "w") as s:
s.close()
with open(AME_CRT, "w") as s:
s.close()
with open(AME_KEY, "w") as s:
s.close()
kube_commands.KUBELET_YAML = kubelet_yaml
kube_commands.CNI_DIR = CNI_DIR
kube_commands.FLANNEL_CONF_FILE = FLANNEL_CONF_FILE
kube_commands.SERVER_ADMIN_URL = "file://{}".format(self.admin_conf_file)
kube_commands.KUBE_ADMIN_CONF = KUBE_ADMIN_CONF
kube_commands.AME_CRT = AME_CRT
kube_commands.AME_KEY = AME_KEY


@patch("kube_commands.subprocess.Popen")
Expand Down Expand Up @@ -295,11 +321,13 @@ def test_write_labels(self, mock_subproc):
json.dumps(labels, indent=4)))
assert False


@patch("kube_commands.requests.get")
@patch("kube_commands.swsscommon.DBConnector")
@patch("kube_commands.swsscommon.Table")
@patch("kube_commands.subprocess.Popen")
def test_join(self, mock_subproc):
def test_join(self, mock_subproc, mock_table, mock_conn, mock_reqget):
self.init()
common_test.set_kube_mock(mock_subproc)
common_test.set_kube_mock(mock_subproc, mock_table, mock_conn, mock_reqget)

for (i, ct_data) in join_test_data.items():
lock_file = ""
Expand Down

0 comments on commit 7152e84

Please sign in to comment.