Skip to content

Commit

Permalink
Revert "Revert "[Kubernetes]: The kube server could be used as http-p…
Browse files Browse the repository at this point in the history
…roxy for docker (sonic-net#7469)" (sonic-net#8023)" (sonic-net#8158)

This reverts commit 7236fa9.

Restore original PR sonic-net#7469
  • Loading branch information
renukamanavalan committed Jul 20, 2021
1 parent 13a6266 commit ed5f86d
Show file tree
Hide file tree
Showing 11 changed files with 371 additions and 11 deletions.
1 change: 0 additions & 1 deletion build_debian.sh
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,6 @@ then
sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install kubelet=${KUBERNETES_VERSION}-00
sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install kubectl=${KUBERNETES_VERSION}-00
sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install kubeadm=${KUBERNETES_VERSION}-00
# kubeadm package auto install kubelet & kubectl
else
echo '[INFO] Skipping Install kubernetes'
fi
Expand Down
23 changes: 23 additions & 0 deletions files/build_templates/sonic_debian_extension.j2
Original file line number Diff line number Diff line change
Expand Up @@ -414,6 +414,10 @@ sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install azure-
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install watchdog==0.10.3

{% if include_kubernetes == "y" %}
# Point to kubelet to /etc/resolv.conf
#
echo 'KUBELET_EXTRA_ARGS="--resolv-conf=/etc/resolv.conf"' | sudo tee -a $FILESYSTEM_ROOT/etc/default/kubelet

# Copy Flannel conf file into sonic-templates
#
sudo cp $BUILD_TEMPLATES/kube_cni.10-flannel.conflist $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/
Expand All @@ -434,6 +438,25 @@ sudo cp ${files_path}/container_startup.py ${FILESYSTEM_ROOT_USR_SHARE_SONIC_SCR
sudo chmod a+x ${FILESYSTEM_ROOT_USR_SHARE_SONIC_SCRIPTS}/container_startup.py

# Config file used by container mgmt scripts/service
fl="${files_path}/remote_ctr.config.json"
use_k8s_as_http_proxy=$(python3 -c 'import json
with open("'${fl}'", "r") as s:
d=json.load(s);print(d.get("use_k8s_as_http_proxy", ""))
')
if [ "${use_k8s_as_http_proxy}" == "y" ]; then
# create proxy files for docker using private IP which will
# be later directed to k8s master upon config
PROXY_INFO="http://172.16.1.1:3128/"
cat <<EOT | sudo tee $FILESYSTEM_ROOT/etc/systemd/system/docker.service.d/http_proxy.conf > /dev/null
[Service]
Environment="HTTP_PROXY=${PROXY_INFO}"
EOT
cat <<EOT | sudo tee $FILESYSTEM_ROOT/etc/systemd/system/docker.service.d/https_proxy.conf > /dev/null
[Service]
Environment="HTTPS_PROXY=${PROXY_INFO}"
EOT
fi

sudo cp ${files_path}/remote_ctr.config.json ${FILESYSTEM_ROOT_ETC_SONIC}/

# Remote container management service files
Expand Down
7 changes: 4 additions & 3 deletions rules/config
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,7 @@ INCLUDE_NAT = y
# TELEMETRY_WRITABLE - Enable write/config operations via the gNMI interface.
# Uncomment to enable:
# TELEMETRY_WRITABLE = y

# INCLUDE_KUBERNETES - if set to y kubernetes packages are installed to be able to
# run as worker node in kubernetes cluster.
INCLUDE_KUBERNETES = n
Expand All @@ -151,9 +152,9 @@ INCLUDE_KUBERNETES = n
# These are Used *only* when INCLUDE_KUBERNETES=y
# NOTE: As a worker node it has to run version compatible to kubernetes master.
#
KUBERNETES_VERSION = 1.18.6
KUBERNETES_CNI_VERSION = 0.8.6
K8s_GCR_IO_PAUSE_VERSION = 3.2
KUBERNETES_VERSION = 1.21.1
KUBERNETES_CNI_VERSION = 0.8.7
K8s_GCR_IO_PAUSE_VERSION = 3.4.1

# SONIC_ENABLE_IMAGE_SIGNATURE - enable image signature
# To not use the auto-generated self-signed certificate, the required files to sign the image as below:
Expand Down
135 changes: 135 additions & 0 deletions src/sonic-ctrmgrd/ctrmgr/ctrmgr_iptables.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
#!/usr/bin/env python3

import ipaddress
import os
import re
import socket
import subprocess
import syslog

UNIT_TESTING = 0

# NOTE:
# Unable to use python-iptables as that does not create rules per ip-tables default
# which is nf_tables. So rules added via iptc package will not be listed under
# "sudo iptables -t nat -L -n". But available in kernel. To list, we need to
# use legacy mode as "sudo iptables-legacy -t nat -L -n".
# As we can't use two modes and using non-default could make any debugging effort
# very tough.


from urllib.parse import urlparse

DST_FILE = "/etc/systemd/system/docker.service.d/http_proxy.conf"
DST_IP = None
DST_PORT = None
SQUID_PORT = "3128"

def _get_ip(ip_str):
ret = ""
if ip_str:
try:
ipaddress.ip_address(ip_str)
ret = ip_str
except ValueError:
pass

if not ret:
try:
ret = socket.gethostbyname(ip_str)
except (OSError, socket.error):
pass
if not ret:
syslog.syslog(syslog.LOG_ERR, "{} is neither IP nor resolves to IP".
format(ip_str))
return ret


def _get_dst_info():
global DST_IP, DST_PORT
DST_IP = None
DST_PORT = None
print("DST_FILE={}".format(DST_FILE))
if os.path.exists(DST_FILE):
with open(DST_FILE, "r") as s:
for line in s.readlines():
url_match = re.search('^Environment=.HTTP_PROXY=(.+?)"', line)
if url_match:
url = urlparse(url_match.group(1))
DST_IP = _get_ip(url.hostname)
DST_PORT = url.port
break
else:
print("{} not available".format(DST_FILE))
print("DST_IP={}".format(DST_IP))


def _is_rule_match(rule):
expect = "DNAT tcp -- 0.0.0.0/0 {} tcp dpt:{} to:".format(
DST_IP, DST_PORT)

# Remove duplicate spaces
rule = " ".join(rule.split()).strip()

if rule.startswith(expect):
return rule[len(expect):]
else:
return ""


def check_proc(proc):
if proc.returncode:
syslog.syslog(syslog.LOG_ERR, "Failed to run: cmd: {}".format(proc.args))
syslog.syslog(syslog.LOG_ERR, "Failed to run: stdout: {}".format(proc.stdout))
syslog.syslog(syslog.LOG_ERR, "Failed to run: stderr: {}".format(proc.stderr))
if not UNIT_TESTING:
assert False


def iptable_proxy_rule_upd(ip_str, port = SQUID_PORT):
_get_dst_info()
if not DST_IP:
# There is no proxy in use. Bail out.
return ""

destination = ""
if ip_str:
upd_ip = _get_ip(ip_str)
if not upd_ip:
return ""
destination = "{}:{}".format(upd_ip, port)

found = False
num = 0

while True:
num += 1

cmd = "sudo iptables -t nat -n -L OUTPUT {}".format(num)
proc = subprocess.run(cmd, shell=True, capture_output=True)
check_proc(proc)

if not proc.stdout:
# No more rule
break

rule_dest = _is_rule_match(proc.stdout.decode("utf-8").strip())
if rule_dest:
if not found and destination and (rule_dest == destination):
found = True
else:
# Duplicate or different IP - delete it
cmd = "sudo iptables -t nat -D OUTPUT {}".format(num)
proc = subprocess.run(cmd, shell=True, capture_output=True)
check_proc(proc)
# Decrement number to accommodate deleted rule
num -= 1

if destination and not found:
cmd = "sudo iptables -t nat -A OUTPUT -p tcp -d {} --dport {} -j DNAT --to-destination {}".format(
DST_IP, DST_PORT, destination)
proc = subprocess.run(cmd, shell=True, capture_output=True)

check_proc(proc)

return destination
13 changes: 11 additions & 2 deletions src/sonic-ctrmgrd/ctrmgr/ctrmgrd.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import syslog

from collections import defaultdict
from ctrmgr.ctrmgr_iptables import iptable_proxy_rule_upd

from swsscommon import swsscommon
from sonic_py_common import device_info
Expand Down Expand Up @@ -87,11 +88,13 @@
JOIN_LATENCY = "join_latency_on_boot_seconds"
JOIN_RETRY = "retry_join_interval_seconds"
LABEL_RETRY = "retry_labels_update_seconds"
USE_K8S_PROXY = "use_k8s_as_http_proxy"

remote_ctr_config = {
JOIN_LATENCY: 10,
JOIN_RETRY: 10,
LABEL_RETRY: 2
LABEL_RETRY: 2,
USE_K8S_PROXY: ""
}

def log_debug(m):
Expand Down Expand Up @@ -309,6 +312,9 @@ def __init__(self, server):

self.start_time = datetime.datetime.now()

if remote_ctr_config[USE_K8S_PROXY] == "y":
iptable_proxy_rule_upd(self.cfg_server[CFG_SER_IP])

if not self.st_server[ST_FEAT_UPDATE_TS]:
# This is upon system start. Sleep 10m before join
self.start_time += datetime.timedelta(
Expand Down Expand Up @@ -336,6 +342,9 @@ def on_config_update(self, key, op, data):
log_debug("Received config update: {}".format(str(data)))
self.cfg_server = cfg_data

if remote_ctr_config[USE_K8S_PROXY] == "y":
iptable_proxy_rule_upd(self.cfg_server[CFG_SER_IP])

if self.pending:
tnow = datetime.datetime.now()
if tnow < self.start_time:
Expand All @@ -359,7 +368,7 @@ def handle_update(self):

ip = self.cfg_server[CFG_SER_IP]
disable = self.cfg_server[CFG_SER_DISABLE] != "false"

pre_state = dict(self.st_server)
log_debug("server: handle_update: disable={} ip={}".format(disable, ip))
if disable or not ip:
Expand Down
9 changes: 5 additions & 4 deletions src/sonic-ctrmgrd/ctrmgr/ctrmgrd.service
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
[Unit]
Description=Container Manager watcher daemon
Requires=updategraph.service
After=updategraph.service

Requires=caclmgrd.service
After=caclmgrd.service
BindsTo=sonic.target
After=sonic.target

[Service]
Type=simple
Expand All @@ -11,4 +12,4 @@ Restart=always
RestartSec=30

[Install]
WantedBy=multi-user.target
WantedBy=sonic.target
3 changes: 2 additions & 1 deletion src/sonic-ctrmgrd/ctrmgr/remote_ctr.config.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
"join_latency_on_boot_seconds": 300,
"retry_join_interval_seconds": 30,
"retry_labels_update_seconds": 5,
"revert_to_local_on_wait_seconds": 60
"revert_to_local_on_wait_seconds": 60,
"use_k8s_as_http_proxy": "y"
}

Loading

0 comments on commit ed5f86d

Please sign in to comment.