Revert "Revert "[Kubernetes]: The kube server could be used as http-proxy for docker""

build_debian.sh

@@ -237,7 +237,6 @@ then
     sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install kubelet=${KUBERNETES_VERSION}-00
     sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install kubectl=${KUBERNETES_VERSION}-00
     sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install kubeadm=${KUBERNETES_VERSION}-00
-    # kubeadm package auto install kubelet & kubectl
 else
     echo '[INFO] Skipping Install kubernetes'
 fi

files/build_templates/sonic_debian_extension.j2

@@ -448,6 +448,10 @@ sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install azure-
 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install watchdog==0.10.3
 
 {% if include_kubernetes == "y" %}
+# Point to kubelet to /etc/resolv.conf
+#
+echo 'KUBELET_EXTRA_ARGS="--resolv-conf=/etc/resolv.conf"' | sudo tee -a  $FILESYSTEM_ROOT/etc/default/kubelet
+
 # Copy Flannel conf file into sonic-templates
 #
 sudo cp $BUILD_TEMPLATES/kube_cni.10-flannel.conflist $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/
@@ -468,6 +472,25 @@ sudo cp ${files_path}/container_startup.py ${FILESYSTEM_ROOT_USR_SHARE_SONIC_SCR
 sudo chmod a+x ${FILESYSTEM_ROOT_USR_SHARE_SONIC_SCRIPTS}/container_startup.py
 
 # Config file used by container mgmt scripts/service
+fl="${files_path}/remote_ctr.config.json"
+use_k8s_as_http_proxy=$(python3 -c 'import json
+with open("'${fl}'", "r") as s:
+    d=json.load(s);print(d.get("use_k8s_as_http_proxy", ""))
+')
+if [ "${use_k8s_as_http_proxy}" == "y" ]; then
+    # create proxy files for docker using private IP which will
+    # be later directed to k8s master upon config
+    PROXY_INFO="http://172.16.1.1:3128/"
+    cat <<EOT | sudo tee $FILESYSTEM_ROOT/etc/systemd/system/docker.service.d/http_proxy.conf > /dev/null
+[Service]
+Environment="HTTP_PROXY=${PROXY_INFO}"
+EOT
+    cat <<EOT | sudo tee $FILESYSTEM_ROOT/etc/systemd/system/docker.service.d/https_proxy.conf > /dev/null
+[Service]
+Environment="HTTPS_PROXY=${PROXY_INFO}"
+EOT
+fi
+
 sudo cp ${files_path}/remote_ctr.config.json ${FILESYSTEM_ROOT_ETC_SONIC}/
 
 # Remote container management service files

rules/config

@@ -142,6 +142,7 @@ INCLUDE_NAT = y
 # TELEMETRY_WRITABLE - Enable write/config operations via the gNMI interface.
 # Uncomment to enable:
 # TELEMETRY_WRITABLE = y
+
 # INCLUDE_KUBERNETES - if set to y kubernetes packages are installed to be able to
 # run as worker node in kubernetes cluster.
 INCLUDE_KUBERNETES = n
@@ -154,9 +155,9 @@ INCLUDE_MACSEC = y
 # These are Used *only* when INCLUDE_KUBERNETES=y
 # NOTE: As a worker node it has to run version compatible to kubernetes master.
 #
-KUBERNETES_VERSION = 1.18.6
-KUBERNETES_CNI_VERSION = 0.8.6
-K8s_GCR_IO_PAUSE_VERSION = 3.2
+KUBERNETES_VERSION = 1.21.1
+KUBERNETES_CNI_VERSION = 0.8.7
+K8s_GCR_IO_PAUSE_VERSION = 3.4.1
 
 # SONIC_ENABLE_IMAGE_SIGNATURE - enable image signature
 # To not use the auto-generated self-signed certificate, the required files to sign the image as below:

src/sonic-ctrmgrd/ctrmgr/ctrmgr_iptables.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+
+import ipaddress
+import os
+import re
+import socket
+import subprocess
+import syslog
+
+UNIT_TESTING = 0
+
+# NOTE:
+# Unable to use python-iptables as that does not create rules per ip-tables default
+# which is nf_tables. So rules added via iptc package will not be listed under
+# "sudo iptables -t nat -L -n". But available in kernel. To list, we need to 
+# use legacy mode as "sudo iptables-legacy -t nat -L -n".
+# As we can't use two modes and using non-default could make any debugging effort
+# very tough.
+
+
+from urllib.parse import urlparse
+
+DST_FILE = "/etc/systemd/system/docker.service.d/http_proxy.conf"
+DST_IP = None
+DST_PORT = None
+SQUID_PORT = "3128"
+
+def _get_ip(ip_str):
+    ret = ""
+    if ip_str:
+        try:
+            ipaddress.ip_address(ip_str)
+            ret = ip_str
+        except ValueError:
+            pass
+
+        if not ret:
+            try:
+                ret = socket.gethostbyname(ip_str)
+            except (OSError, socket.error):
+                pass
+        if not ret:
+            syslog.syslog(syslog.LOG_ERR, "{} is neither IP nor resolves to IP".
+                    format(ip_str))
+    return ret
+
+
+def _get_dst_info():
+    global DST_IP, DST_PORT
+    DST_IP = None
+    DST_PORT = None
+    print("DST_FILE={}".format(DST_FILE))
+    if os.path.exists(DST_FILE):
+        with open(DST_FILE, "r") as s:
+            for line in s.readlines():
+                url_match = re.search('^Environment=.HTTP_PROXY=(.+?)"', line)
+                if url_match:
+                    url = urlparse(url_match.group(1))
+                    DST_IP = _get_ip(url.hostname)
+                    DST_PORT = url.port
+                    break
+    else:
+        print("{} not available".format(DST_FILE))
+    print("DST_IP={}".format(DST_IP))
+
+
+def _is_rule_match(rule):
+    expect = "DNAT tcp -- 0.0.0.0/0 {} tcp dpt:{} to:".format(
+            DST_IP, DST_PORT)
+
+    # Remove duplicate spaces
+    rule = " ".join(rule.split()).strip()
+
+    if rule.startswith(expect):
+        return rule[len(expect):]
+    else:
+        return ""
+
+
+def check_proc(proc):
+    if proc.returncode:
+        syslog.syslog(syslog.LOG_ERR, "Failed to run: cmd: {}".format(proc.args))
+        syslog.syslog(syslog.LOG_ERR, "Failed to run: stdout: {}".format(proc.stdout))
+        syslog.syslog(syslog.LOG_ERR, "Failed to run: stderr: {}".format(proc.stderr))
+        if not UNIT_TESTING:
+            assert False
+
+
+def iptable_proxy_rule_upd(ip_str, port = SQUID_PORT):
+    _get_dst_info()
+    if not DST_IP:
+        # There is no proxy in use. Bail out.
+        return ""
+
+    destination = ""
+    if ip_str:
+        upd_ip = _get_ip(ip_str)
+        if not upd_ip:
+            return ""
+        destination = "{}:{}".format(upd_ip, port)
+
+    found = False
+    num = 0
+
+    while True:
+        num += 1
+
+        cmd = "sudo iptables -t nat -n -L OUTPUT {}".format(num)
+        proc = subprocess.run(cmd, shell=True, capture_output=True)
+        check_proc(proc)
+
+        if not proc.stdout:
+            # No more rule
+            break
+
+        rule_dest = _is_rule_match(proc.stdout.decode("utf-8").strip())
+        if rule_dest:
+            if not found and destination and (rule_dest == destination):
+                found = True
+            else:
+                # Duplicate or different IP - delete it
+                cmd = "sudo iptables -t nat -D OUTPUT {}".format(num)
+                proc = subprocess.run(cmd, shell=True, capture_output=True)
+                check_proc(proc)
+                # Decrement number to accommodate deleted rule
+                num -= 1
+
+    if destination and not found:
+        cmd = "sudo iptables -t nat -A OUTPUT -p tcp -d {} --dport {} -j DNAT --to-destination {}".format(
+                DST_IP, DST_PORT, destination)
+        proc = subprocess.run(cmd, shell=True, capture_output=True)
+
+        check_proc(proc)
+
+    return destination

src/sonic-ctrmgrd/ctrmgr/ctrmgrd.py

@@ -8,6 +8,7 @@
 import syslog
 
 from collections import defaultdict
+from ctrmgr.ctrmgr_iptables import iptable_proxy_rule_upd
 
 from swsscommon import swsscommon
 from sonic_py_common import device_info
@@ -87,11 +88,13 @@
 JOIN_LATENCY = "join_latency_on_boot_seconds"
 JOIN_RETRY = "retry_join_interval_seconds"
 LABEL_RETRY = "retry_labels_update_seconds"
+USE_K8S_PROXY = "use_k8s_as_http_proxy"
 
 remote_ctr_config = {
     JOIN_LATENCY: 10,
     JOIN_RETRY: 10,
-    LABEL_RETRY: 2
+    LABEL_RETRY: 2,
+    USE_K8S_PROXY: ""
     }
 
 def log_debug(m):
@@ -309,6 +312,9 @@ def __init__(self, server):
 
         self.start_time = datetime.datetime.now()
 
+        if remote_ctr_config[USE_K8S_PROXY] == "y":
+            iptable_proxy_rule_upd(self.cfg_server[CFG_SER_IP])
+
         if not self.st_server[ST_FEAT_UPDATE_TS]:
             # This is upon system start. Sleep 10m before join
             self.start_time += datetime.timedelta(
@@ -336,6 +342,9 @@ def on_config_update(self, key, op, data):
         log_debug("Received config update: {}".format(str(data)))
         self.cfg_server = cfg_data
 
+        if remote_ctr_config[USE_K8S_PROXY] == "y":
+            iptable_proxy_rule_upd(self.cfg_server[CFG_SER_IP])
+
         if self.pending:
             tnow = datetime.datetime.now()
             if tnow < self.start_time:
@@ -359,7 +368,7 @@ def handle_update(self):
 
         ip = self.cfg_server[CFG_SER_IP]
         disable = self.cfg_server[CFG_SER_DISABLE] != "false"
-        
+
         pre_state = dict(self.st_server)
         log_debug("server: handle_update: disable={} ip={}".format(disable, ip))
         if disable or not ip:

src/sonic-ctrmgrd/ctrmgr/ctrmgrd.service

@@ -1,8 +1,9 @@
 [Unit]
 Description=Container Manager watcher daemon
-Requires=updategraph.service
-After=updategraph.service
-
+Requires=caclmgrd.service
+After=caclmgrd.service
+BindsTo=sonic.target
+After=sonic.target
 
 [Service]
 Type=simple
@@ -11,4 +12,4 @@ Restart=always
 RestartSec=30
 
 [Install]
-WantedBy=multi-user.target
+WantedBy=sonic.target

src/sonic-ctrmgrd/ctrmgr/remote_ctr.config.json

@@ -2,6 +2,7 @@
     "join_latency_on_boot_seconds": 300,
     "retry_join_interval_seconds": 30,
     "retry_labels_update_seconds": 5,
-    "revert_to_local_on_wait_seconds": 60
+    "revert_to_local_on_wait_seconds": 60,
+    "use_k8s_as_http_proxy": "y"
 }