From 59caca1059bc72701ea00ef3be926fe8fa7fa4dc Mon Sep 17 00:00:00 2001 From: Kiran Kella Date: Fri, 20 Sep 2019 04:57:06 -0700 Subject: [PATCH 1/7] Changes in sonic-buildimage for the NAT feature - Docker for NAT - installing the required tools iptables and conntrack for nat Signed-off-by: kiran.kella@broadcom.com --- build_debian.sh | 3 +- dockers/docker-nat/Dockerfile.j2 | 37 +++ dockers/docker-nat/base_image_files/natctl | 5 + dockers/docker-nat/restore_nat_entries.py | 96 +++++++ dockers/docker-nat/start.sh | 17 ++ dockers/docker-nat/supervisord.conf | 47 +++ dockers/docker-orchagent/Dockerfile.j2 | 3 +- files/build_templates/nat.service.j2 | 15 + .../build_templates/sonic_debian_extension.j2 | 4 + rules/docker-nat.mk | 30 ++ rules/iptables.mk | 27 ++ sonic-slave-stretch/Dockerfile.j2 | 3 + src/iptables/Makefile | 47 +++ ...ng-fullcone-option-for-SNAT-and-DNAT.patch | 267 ++++++++++++++++++ src/iptables/patch/series | 1 + 15 files changed, 600 insertions(+), 2 deletions(-) create mode 100644 dockers/docker-nat/Dockerfile.j2 create mode 100644 dockers/docker-nat/base_image_files/natctl create mode 100755 dockers/docker-nat/restore_nat_entries.py create mode 100755 dockers/docker-nat/start.sh create mode 100644 dockers/docker-nat/supervisord.conf create mode 100644 files/build_templates/nat.service.j2 create mode 100644 rules/docker-nat.mk create mode 100644 rules/iptables.mk create mode 100644 src/iptables/Makefile create mode 100644 src/iptables/patch/0001-Passing-fullcone-option-for-SNAT-and-DNAT.patch create mode 100644 src/iptables/patch/series diff --git a/build_debian.sh b/build_debian.sh index 666e140416c9..0db587e434c7 100755 --- a/build_debian.sh +++ b/build_debian.sh @@ -273,7 +273,8 @@ sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y in cgroup-tools \ ipmitool \ ndisc6 \ - makedumpfile + makedumpfile \ + conntrack if [[ $CONFIGURED_ARCH == amd64 ]]; then diff --git a/dockers/docker-nat/Dockerfile.j2 b/dockers/docker-nat/Dockerfile.j2 new file mode 100644 index 000000000000..75dc0e1c842d --- /dev/null +++ b/dockers/docker-nat/Dockerfile.j2 @@ -0,0 +1,37 @@ +FROM {{ docker_nat_load_image }} + +ARG docker_container_name +RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf + +RUN echo + +## Make apt-get non-interactive +ENV DEBIAN_FRONTEND=noninteractive + +## Install redis-tools dependencies +## TODO: implicitly install dependencies +RUN apt-get update \ + && apt-get install -f -y libdbus-1-3 libdaemon0 libjansson4 libpython2.7 \ + && apt-get -y install libjemalloc1 \ + && apt-get -y install conntrack + +COPY \ +{% for deb in docker_nat_debs.split(' ') -%} +debs/{{ deb }}{{' '}} +{%- endfor -%} +debs/ + +RUN dpkg -i \ +{% for deb in docker_nat_debs.split(' ') -%} +debs/{{ deb }}{{' '}} +{%- endfor %} + +COPY ["start.sh", "/usr/bin/"] +COPY ["supervisord.conf", "/etc/supervisor/conf.d/"] +COPY ["restore_nat_entries.py", "/usr/bin/"] + +RUN apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y +RUN rm -rf /debs + +ENTRYPOINT ["/usr/bin/supervisord"] + diff --git a/dockers/docker-nat/base_image_files/natctl b/dockers/docker-nat/base_image_files/natctl new file mode 100644 index 000000000000..6cba3c86be70 --- /dev/null +++ b/dockers/docker-nat/base_image_files/natctl @@ -0,0 +1,5 @@ +#!/bin/bash + +# -t option needed only for shell, not for commands + +docker exec -i nat natctl "$@" diff --git a/dockers/docker-nat/restore_nat_entries.py b/dockers/docker-nat/restore_nat_entries.py new file mode 100755 index 000000000000..a6488495f0da --- /dev/null +++ b/dockers/docker-nat/restore_nat_entries.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python + +"""" +Description: restore_nat_entries.py -- restoring nat entries table into kernel during system warm reboot. + The script is started by supervisord in nat docker when the docker is started. + It does not do anything in case neither system nor nat warm restart is enabled. + In case nat warm restart enabled only, it sets the stateDB flag so natsyncd can continue + the reconciation process. + In case system warm reboot is enabled, it will try to restore the nat entries table into kernel + , then it sets the stateDB flag for natsyncd to continue the + reconciliation process. +""" + +import sys +import subprocess +import swsssdk +from swsscommon import swsscommon +import logging +import re +import os + +logging.getLogger("scapy.runtime").setLevel(logging.ERROR) + +logger = logging.getLogger(__name__) +logger.setLevel(logging.WARNING) +logger.addHandler(logging.NullHandler()) + +def add_nat_conntrack_entry_in_kernel(ipproto, srcip, dstip, srcport, dstport, natsrcip, natdstip, natsrcport, natdstport): + # pyroute2 doesn't have support for adding conntrack entries via netlink yet. So, invoking the conntrack utility to add the entries. + state = '' + if (ipproto == '6'): + state = ' --state ESTABLISHED ' + ctcmd = 'conntrack -I -n ' + natdstip + ':' + natdstport + ' -g ' + natsrcip + ':' + natsrcport + \ + ' --protonum ' + ipproto + state + ' --timeout 600 --src ' + srcip + ' --sport ' + srcport + \ + ' --dst ' + dstip + ' --dport ' + dstport + ' -u ASSURED' + subprocess.call(ctcmd, shell=True) + +# Set the statedb "NAT_RESTORE_TABLE|Flags", so natsyncd can start reconciliation +def set_statedb_nat_restore_done(): + db = swsssdk.SonicV2Connector(host='127.0.0.1') + db.connect(db.STATE_DB, False) + db.set(db.STATE_DB, 'NAT_RESTORE_TABLE|Flags', 'restored', 'true') + db.close(db.STATE_DB) + return + +# This function is to restore the kernel nat entries based on the saved nat entries. +def restore_update_kernel_nat_entries(filename): + # Read the entries from nat_entries.dump file and add them to kernel + with open(filename, 'r') as fp: + for line in fp: + ctline = re.findall(r'^(\w+)\s+(\d+).*src=([\d.]+)\s+dst=([\d.]+)\s+sport=(\d+)\s+dport=(\d+).*src=([\d.]+)\s+dst=([\d.]+)\s+sport=(\d+)\s+dport=(\d+)', line) + if not ctline: + continue + cmdargs = ctline[0] + if ((cmdargs[0] != 'tcp') and (cmdargs[0] != 'udp')): + continue + add_nat_conntrack_entry_in_kernel(cmdargs[1], cmdargs[2], cmdargs[3], cmdargs[4], cmdargs[5], cmdargs[6], cmdargs[7], cmdargs[8], cmdargs[9]) + +def main(): + + print("restore_nat_entries service is started") + + # Use warmstart python binding to check warmstart information + warmstart = swsscommon.WarmStart() + warmstart.initialize("natsyncd", "nat") + warmstart.checkWarmStart("natsyncd", "nat", False) + + # if swss or system warm reboot not enabled, don't run + if not warmstart.isWarmStart(): + print("restore_nat_entries service is skipped as warm restart not enabled") + return + + # NAT restart not system warm reboot, set statedb directly + if not warmstart.isSystemWarmRebootEnabled(): + set_statedb_nat_restore_done() + print("restore_nat_entries service is done as system warm reboot not enabled") + return + + # Program the nat conntrack entries in the kernel by reading the + # entries from nat_entries.dump + try: + restore_update_kernel_nat_entries('/var/warmboot/nat/nat_entries.dump') + except Exception as e: + logger.exception(str(e)) + sys.exit(1) + + # Remove the dump file after restoration + os.remove('/var/warmboot/nat/nat_entries.dump') + + # set statedb to signal other processes like natsyncd + set_statedb_nat_restore_done() + print("restore_nat_entries service is done for system warmreboot") + return + +if __name__ == '__main__': + main() diff --git a/dockers/docker-nat/start.sh b/dockers/docker-nat/start.sh new file mode 100755 index 000000000000..0d89d990d29e --- /dev/null +++ b/dockers/docker-nat/start.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +rm -f /var/run/rsyslogd.pid +rm -f /var/run/nat/* + +mkdir -p /var/warmboot/nat + +supervisorctl start rsyslogd + +supervisorctl start natmgrd + +sleep 5 + +supervisorctl start natsyncd + +supervisorctl start restore_nat_entries + diff --git a/dockers/docker-nat/supervisord.conf b/dockers/docker-nat/supervisord.conf new file mode 100644 index 000000000000..bb42d23fe355 --- /dev/null +++ b/dockers/docker-nat/supervisord.conf @@ -0,0 +1,47 @@ +[supervisord] +logfile_maxbytes=1MB +logfile_backups=2 +nodaemon=true + +[program:start.sh] +command=/usr/bin/start.sh +priority=1 +autostart=true +autorestart=false +stdout_logfile=syslog +stderr_logfile=syslog + +[program:rsyslogd] +command=/usr/sbin/rsyslogd -n +priority=2 +autostart=false +autorestart=false +stdout_logfile=syslog +stderr_logfile=syslog + +[program:natmgrd] +command=/usr/bin/natmgrd +priority=3 +autostart=false +autorestart=false +stdout_logfile=syslog +stderr_logfile=syslog + +[program:natsyncd] +command=/usr/bin/natsyncd +priority=4 +autostart=false +autorestart=false +stdout_logfile=syslog +stderr_logfile=syslog + +[program:restore_nat_entries] +command=/usr/bin/restore_nat_entries.py +priority=5 +autostart=false +autorestart=false +startsecs=0 +startretries=0 +stdout_logfile=syslog +stderr_logfile=syslog + diff --git a/dockers/docker-orchagent/Dockerfile.j2 b/dockers/docker-orchagent/Dockerfile.j2 index 8a66e2adbe43..f95acd48fdbd 100755 --- a/dockers/docker-orchagent/Dockerfile.j2 +++ b/dockers/docker-orchagent/Dockerfile.j2 @@ -20,7 +20,8 @@ RUN apt-get update && \ tcpdump \ libelf1 \ libmnl0 \ - bridge-utils + bridge-utils \ + conntrack {% if ( CONFIGURED_ARCH == "armhf" or CONFIGURED_ARCH == "arm64" ) %} ## Fix for gcc/python not found in arm docker diff --git a/files/build_templates/nat.service.j2 b/files/build_templates/nat.service.j2 new file mode 100644 index 000000000000..2e3e17439ef7 --- /dev/null +++ b/files/build_templates/nat.service.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=NAT container +Requires=updategraph.service swss.service +After=updategraph.service swss.service syncd.service +Before=ntp-config.service + +[Service] +User={{ sonicadmin_user }} +ExecStartPre=/usr/bin/{{docker_container_name}}.sh start +ExecStart=/usr/bin/{{docker_container_name}}.sh wait +ExecStop=/usr/bin/{{docker_container_name}}.sh stop + +[Install] +WantedBy=multi-user.target swss.service + diff --git a/files/build_templates/sonic_debian_extension.j2 b/files/build_templates/sonic_debian_extension.j2 index b63b5addac4b..0fa63a9fca8b 100644 --- a/files/build_templates/sonic_debian_extension.j2 +++ b/files/build_templates/sonic_debian_extension.j2 @@ -81,6 +81,10 @@ sudo mkdir -p $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/ sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/ifupdown2_*.deb || \ sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f +# Install ipables (and its dependencies via 'apt-get -y install -f') +sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/iptables_*.deb || \ + sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f + # Install dependencies for SONiC config engine sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install \ python-dev \ diff --git a/rules/docker-nat.mk b/rules/docker-nat.mk new file mode 100644 index 000000000000..765b6f7d3b0f --- /dev/null +++ b/rules/docker-nat.mk @@ -0,0 +1,30 @@ +# docker image for nat + +DOCKER_NAT_STEM = docker-nat +DOCKER_NAT = $(DOCKER_NAT_STEM).gz +DOCKER_NAT_DBG = $(DOCKER_NAT_STEM)-$(DBG_IMAGE_MARK).gz + +$(DOCKER_NAT)_PATH = $(DOCKERS_PATH)/$(DOCKER_NAT_STEM) + +$(DOCKER_NAT)_DEPENDS += $(SWSS) $(REDIS_TOOLS) $(IPTABLESIP4TC) $(IPTABLESIP6TC) $(IPTABLESIPTC) $(IPXTABLES12) $(IPTABLES) +$(DOCKER_NAT)_DBG_DEPENDS = $($(DOCKER_CONFIG_ENGINE_STRETCH)_DBG_DEPENDS) +$(DOCKER_NAT)_DBG_DEPENDS += $(SWSS_DBG) $(LIBSWSSCOMMON_DBG) +$(DOCKER_NAT)_DBG_IMAGE_PACKAGES = $($(DOCKER_CONFIG_ENGINE_STRETCH)_DBG_IMAGE_PACKAGES) + +$(DOCKER_NAT)_LOAD_DOCKERS += $(DOCKER_CONFIG_ENGINE_STRETCH) + +SONIC_DOCKER_IMAGES += $(DOCKER_NAT) +SONIC_INSTALL_DOCKER_IMAGES += $(DOCKER_NAT) +SONIC_STRETCH_DOCKERS += $(DOCKER_NAT) + +SONIC_DOCKER_DBG_IMAGES += $(DOCKER_NAT_DBG) +SONIC_INSTALL_DOCKER_DBG_IMAGES += $(DOCKER_NAT_DBG) +SONIC_STRETCH_DBG_DOCKERS += $(DOCKER_NAT_DBG) + +$(DOCKER_NAT)_CONTAINER_NAME = nat +$(DOCKER_NAT)_RUN_OPT += --net=host --privileged -t +$(DOCKER_NAT)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro +$(DOCKER_NAT)_RUN_OPT += -v /host/warmboot:/var/warmboot + +$(DOCKER_NAT)_BASE_IMAGE_FILES += natctl:/usr/bin/natctl + diff --git a/rules/iptables.mk b/rules/iptables.mk new file mode 100644 index 000000000000..4d88c0a224b2 --- /dev/null +++ b/rules/iptables.mk @@ -0,0 +1,27 @@ +# iptables package + +IPTABLES_VERSION = 1.6.0+snapshot20161117 +IPTABLES_VERSION_SUFFIX = 6 +IPTABLES_VERSION_FULL = $(IPTABLES_VERSION)-$(IPTABLES_VERSION_SUFFIX) + +IPTABLES = iptables_$(IPTABLES_VERSION_FULL)_amd64.deb +$(IPTABLES)_SRC_PATH = $(SRC_PATH)/iptables +SONIC_MAKE_DEBS += $(IPTABLES) +SONIC_STRETCH_DEBS += $(IPTABLES) + +IPTABLESIP4TC = libip4tc0_$(IPTABLES_VERSION_FULL)_amd64.deb +$(eval $(call add_derived_package,$(IPTABLES),$(IPTABLESIP4TC))) + +IPTABLESIP6TC = libip6tc0_$(IPTABLES_VERSION_FULL)_amd64.deb +$(eval $(call add_derived_package,$(IPTABLES),$(IPTABLESIP6TC))) + +IPTABLESIPTC = libiptc0_$(IPTABLES_VERSION_FULL)_amd64.deb +$(eval $(call add_derived_package,$(IPTABLES),$(IPTABLESIPTC))) + +IPXTABLES12 = libxtables12_$(IPTABLES_VERSION_FULL)_amd64.deb +$(eval $(call add_derived_package,$(IPTABLES),$(IPXTABLES12))) + +# Export these variables so they can be used in a sub-make +export IPTABLES_VERSION +export IPTABLES_VERSION_FULL +export IPTABLES diff --git a/sonic-slave-stretch/Dockerfile.j2 b/sonic-slave-stretch/Dockerfile.j2 index 08e002624abb..28007015e82c 100644 --- a/sonic-slave-stretch/Dockerfile.j2 +++ b/sonic-slave-stretch/Dockerfile.j2 @@ -292,6 +292,9 @@ RUN apt-get update && apt-get install -y \ libselinux1-dev \ # For kdump-tools liblzo2-dev \ +# For iptables + libnetfilter-conntrack-dev \ + libnftnl-dev \ # For SAI3.7 libprotobuf-dev diff --git a/src/iptables/Makefile b/src/iptables/Makefile new file mode 100644 index 000000000000..60154c19ddb1 --- /dev/null +++ b/src/iptables/Makefile @@ -0,0 +1,47 @@ +.ONESHELL: +SHELL = /bin/bash +.SHELLFLAGS += -e + +MAIN_TARGET = $(IPTABLES) +DERIVED_TARGETS = libip4tc0_$(IPTABLES_VERSION_FULL)_amd64.deb \ + libip6tc0_$(IPTABLES_VERSION_FULL)_amd64.deb \ + libiptc0_$(IPTABLES_VERSION_FULL)_amd64.deb \ + libxtables12_$(IPTABLES_VERSION_FULL)_amd64.deb + +IPTABLES_URL = http://deb.debian.org/debian/pool/main/i/iptables + +DSC_FILE = iptables_$(IPTABLES_VERSION_FULL).dsc +ORIG_FILE = iptables_$(IPTABLES_VERSION).orig.tar.bz2 +DEBIAN_FILE = iptables_$(IPTABLES_VERSION_FULL).debian.tar.xz + +DSC_FILE_URL = $(IPTABLES_URL)/$(DSC_FILE) +ORIG_FILE_URL = $(IPTABLES_URL)/$(ORIG_FILE) +DEBIAN_FILE_URL = $(IPTABLES_URL)/$(DEBIAN_FILE) + +$(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : + # Remove any stale files + rm -rf ./iptables-$(IPTABLES_VERSION) + + # Get iptables release + wget -NO "$(DSC_FILE)" $(DSC_FILE_URL) + wget -NO "$(ORIG_FILE)" $(ORIG_FILE_URL) + wget -NO "$(DEBIAN_FILE)" $(DEBIAN_FILE_URL) + dpkg-source -x iptables_$(IPTABLES_VERSION_FULL).dsc + + pushd iptables-$(IPTABLES_VERSION) + git init + git add -f * + git commit -m "unmodified iptables source" + + # Apply patches + stg init + stg import -s ../patch/series + + # Build source and Debian packages + dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) + popd + + # Move the newly-built .deb packages to the destination directory + mv $(DERIVED_TARGETS) $* $(DEST)/ + +$(addprefix $(DEST)/, $(DERIVED_TARGETS)): $(DEST)/% : $(DEST)/$(MAIN_TARGET) diff --git a/src/iptables/patch/0001-Passing-fullcone-option-for-SNAT-and-DNAT.patch b/src/iptables/patch/0001-Passing-fullcone-option-for-SNAT-and-DNAT.patch new file mode 100644 index 000000000000..f7fba85a270b --- /dev/null +++ b/src/iptables/patch/0001-Passing-fullcone-option-for-SNAT-and-DNAT.patch @@ -0,0 +1,267 @@ +From 92f5aee7372748845f11b7a10d880f968769e860 Mon Sep 17 00:00:00 2001 +From: Kiran Kella +Date: Wed, 7 Aug 2019 07:22:42 -0700 +Subject: [PATCH] Passing fullcone option for SNAT and DNAT + +--- + extensions/libipt_DNAT.c | 22 +++++++++++++++++++++- + extensions/libipt_MASQUERADE.c | 21 ++++++++++++++++++++- + extensions/libipt_SNAT.c | 22 +++++++++++++++++++++- + 3 files changed, 62 insertions(+), 3 deletions(-) + +diff --git a/extensions/libipt_DNAT.c b/extensions/libipt_DNAT.c +index a14d16f..4bfab98 100644 +--- a/extensions/libipt_DNAT.c ++++ b/extensions/libipt_DNAT.c +@@ -8,14 +8,20 @@ + #include + #include + ++/* Temporarily defining here, need to be picked up from the ++ * new kernel header linux/netfilter/nf_nat.h */ ++#define NF_NAT_RANGE_FULLCONE (1 << 5) ++ + enum { + O_TO_DEST = 0, + O_RANDOM, + O_PERSISTENT, + O_X_TO_DEST, /* hidden flag */ ++ O_FULLCONE, + F_TO_DEST = 1 << O_TO_DEST, + F_RANDOM = 1 << O_RANDOM, + F_X_TO_DEST = 1 << O_X_TO_DEST, ++ F_FULLCONE = 1 << O_FULLCONE + }; + + /* Dest NAT data consists of a multi-range, indicating where to map +@@ -32,7 +38,7 @@ static void DNAT_help(void) + "DNAT target options:\n" + " --to-destination [[-]][:port[-port]]\n" + " Address to map destination to.\n" +-"[--random] [--persistent]\n"); ++"[--random] [--persistent] [--fullcone]\n"); + } + + static const struct xt_option_entry DNAT_opts[] = { +@@ -40,6 +46,7 @@ static const struct xt_option_entry DNAT_opts[] = { + .flags = XTOPT_MAND | XTOPT_MULTI}, + {.name = "random", .id = O_RANDOM, .type = XTTYPE_NONE}, + {.name = "persistent", .id = O_PERSISTENT, .type = XTTYPE_NONE}, ++ {.name = "fullcone", .id = O_FULLCONE, .type = XTTYPE_NONE}, + XTOPT_TABLEEND, + }; + +@@ -185,10 +192,14 @@ static void DNAT_parse(struct xt_option_call *cb) + static void DNAT_fcheck(struct xt_fcheck_call *cb) + { + static const unsigned int f = F_TO_DEST | F_RANDOM; ++ static const unsigned int c = F_FULLCONE; + struct nf_nat_ipv4_multi_range_compat *mr = cb->data; + + if ((cb->xflags & f) == f) + mr->range[0].flags |= NF_NAT_RANGE_PROTO_RANDOM; ++ ++ if ((cb->xflags & c) == c) ++ mr->range[0].flags |= NF_NAT_RANGE_FULLCONE; + } + + static void print_range(const struct nf_nat_ipv4_range *r) +@@ -224,6 +235,8 @@ static void DNAT_print(const void *ip, const struct xt_entry_target *target, + printf(" random"); + if (info->mr.range[i].flags & NF_NAT_RANGE_PERSISTENT) + printf(" persistent"); ++ if (info->mr.range[i].flags & NF_NAT_RANGE_FULLCONE) ++ printf(" fullcone"); + } + } + +@@ -239,6 +252,8 @@ static void DNAT_save(const void *ip, const struct xt_entry_target *target) + printf(" --random"); + if (info->mr.range[i].flags & NF_NAT_RANGE_PERSISTENT) + printf(" --persistent"); ++ if (info->mr.range[i].flags & NF_NAT_RANGE_FULLCONE) ++ printf(" --fullcone"); + } + } + +@@ -282,6 +297,11 @@ static int DNAT_xlate(struct xt_xlate *xl, + sep = ","; + xt_xlate_add(xl, "%spersistent", sep); + } ++ if (info->mr.range[i].flags & NF_NAT_RANGE_FULLCONE) { ++ if (sep_need) ++ sep = ","; ++ xt_xlate_add(xl, "%sfullcone", sep); ++ } + } + + return 1; +diff --git a/extensions/libipt_MASQUERADE.c b/extensions/libipt_MASQUERADE.c +index b7b5fc7..88ff650 100644 +--- a/extensions/libipt_MASQUERADE.c ++++ b/extensions/libipt_MASQUERADE.c +@@ -8,9 +8,15 @@ + #include + #include + ++/* Temporarily defining here, need to be picked up from the ++ * new kernel header linux/netfilter/nf_nat.h */ ++#define NF_NAT_RANGE_FULLCONE (1 << 5) ++ + enum { + O_TO_PORTS = 0, + O_RANDOM, ++ O_RANDOM_FULLY, ++ O_FULLCONE + }; + + static void MASQUERADE_help(void) +@@ -20,12 +26,15 @@ static void MASQUERADE_help(void) + " --to-ports [-]\n" + " Port (range) to map to.\n" + " --random\n" +-" Randomize source port.\n"); ++" Randomize source port.\n" ++" --fullcone\n" ++" Do fullcone NAT mapping.\n"); + } + + static const struct xt_option_entry MASQUERADE_opts[] = { + {.name = "to-ports", .id = O_TO_PORTS, .type = XTTYPE_STRING}, + {.name = "random", .id = O_RANDOM, .type = XTTYPE_NONE}, ++ {.name = "fullcone", .id = O_FULLCONE, .type = XTTYPE_NONE}, + XTOPT_TABLEEND, + }; + +@@ -97,6 +106,9 @@ static void MASQUERADE_parse(struct xt_option_call *cb) + case O_RANDOM: + mr->range[0].flags |= NF_NAT_RANGE_PROTO_RANDOM; + break; ++ case O_FULLCONE: ++ mr->range[0].flags |= NF_NAT_RANGE_FULLCONE; ++ break; + } + } + +@@ -116,6 +128,8 @@ MASQUERADE_print(const void *ip, const struct xt_entry_target *target, + + if (r->flags & NF_NAT_RANGE_PROTO_RANDOM) + printf(" random"); ++ if (r->flags & NF_NAT_RANGE_FULLCONE) ++ printf(" fullcone"); + } + + static void +@@ -132,6 +146,8 @@ MASQUERADE_save(const void *ip, const struct xt_entry_target *target) + + if (r->flags & NF_NAT_RANGE_PROTO_RANDOM) + printf(" --random"); ++ if (r->flags & NF_NAT_RANGE_FULLCONE) ++ printf(" --fullcone"); + } + + static int MASQUERADE_xlate(struct xt_xlate *xl, +@@ -153,6 +169,9 @@ static int MASQUERADE_xlate(struct xt_xlate *xl, + if (r->flags & NF_NAT_RANGE_PROTO_RANDOM) + xt_xlate_add(xl, "random "); + ++ if (r->flags & NF_NAT_RANGE_FULLCONE) ++ xt_xlate_add(xl, "fullcone "); ++ + return 1; + } + +diff --git a/extensions/libipt_SNAT.c b/extensions/libipt_SNAT.c +index e92d811..9634ba9 100644 +--- a/extensions/libipt_SNAT.c ++++ b/extensions/libipt_SNAT.c +@@ -8,16 +8,22 @@ + #include + #include + ++/* Temporarily defining here, need to be picked up from the ++ * new kernel header linux/netfilter/nf_nat.h */ ++#define NF_NAT_RANGE_FULLCONE (1 << 5) ++ + enum { + O_TO_SRC = 0, + O_RANDOM, + O_RANDOM_FULLY, + O_PERSISTENT, + O_X_TO_SRC, ++ O_FULLCONE, + F_TO_SRC = 1 << O_TO_SRC, + F_RANDOM = 1 << O_RANDOM, + F_RANDOM_FULLY = 1 << O_RANDOM_FULLY, + F_X_TO_SRC = 1 << O_X_TO_SRC, ++ F_FULLCONE = 1 << O_FULLCONE + }; + + /* Source NAT data consists of a multi-range, indicating where to map +@@ -34,7 +40,7 @@ static void SNAT_help(void) + "SNAT target options:\n" + " --to-source [[-]][:port[-port]]\n" + " Address to map source to.\n" +-"[--random] [--random-fully] [--persistent]\n"); ++"[--random] [--random-fully] [--persistent] [--fullcone]\n"); + } + + static const struct xt_option_entry SNAT_opts[] = { +@@ -43,6 +49,7 @@ static const struct xt_option_entry SNAT_opts[] = { + {.name = "random", .id = O_RANDOM, .type = XTTYPE_NONE}, + {.name = "random-fully", .id = O_RANDOM_FULLY, .type = XTTYPE_NONE}, + {.name = "persistent", .id = O_PERSISTENT, .type = XTTYPE_NONE}, ++ {.name = "fullcone", .id = O_FULLCONE, .type = XTTYPE_NONE}, + XTOPT_TABLEEND, + }; + +@@ -189,12 +196,15 @@ static void SNAT_fcheck(struct xt_fcheck_call *cb) + { + static const unsigned int f = F_TO_SRC | F_RANDOM; + static const unsigned int r = F_TO_SRC | F_RANDOM_FULLY; ++ static const unsigned int c = F_TO_SRC | F_FULLCONE; + struct nf_nat_ipv4_multi_range_compat *mr = cb->data; + + if ((cb->xflags & f) == f) + mr->range[0].flags |= NF_NAT_RANGE_PROTO_RANDOM; + if ((cb->xflags & r) == r) + mr->range[0].flags |= NF_NAT_RANGE_PROTO_RANDOM_FULLY; ++ if ((cb->xflags & c) == c) ++ mr->range[0].flags |= NF_NAT_RANGE_FULLCONE; + } + + static void print_range(const struct nf_nat_ipv4_range *r) +@@ -232,6 +242,8 @@ static void SNAT_print(const void *ip, const struct xt_entry_target *target, + printf(" random-fully"); + if (info->mr.range[i].flags & NF_NAT_RANGE_PERSISTENT) + printf(" persistent"); ++ if (info->mr.range[i].flags & NF_NAT_RANGE_FULLCONE) ++ printf(" fullcone"); + } + } + +@@ -249,6 +261,8 @@ static void SNAT_save(const void *ip, const struct xt_entry_target *target) + printf(" --random-fully"); + if (info->mr.range[i].flags & NF_NAT_RANGE_PERSISTENT) + printf(" --persistent"); ++ if (info->mr.range[i].flags & NF_NAT_RANGE_FULLCONE) ++ printf(" --fullcone"); + } + } + +@@ -299,6 +313,12 @@ static int SNAT_xlate(struct xt_xlate *xl, + sep = ","; + xt_xlate_add(xl, "%spersistent", sep); + } ++ if (info->mr.range[i].flags & NF_NAT_RANGE_FULLCONE) { ++ if (sep_need) ++ sep = ","; ++ xt_xlate_add(xl, "%sfullcone", sep); ++ sep_need = true; ++ } + } + + return 1; +-- +2.18.0 + diff --git a/src/iptables/patch/series b/src/iptables/patch/series new file mode 100644 index 000000000000..df084ed96ed3 --- /dev/null +++ b/src/iptables/patch/series @@ -0,0 +1 @@ +0001-Passing-fullcone-option-for-SNAT-and-DNAT.patch From 40fcf2536e77294674eefc0b1e2f5c80a8d100b4 Mon Sep 17 00:00:00 2001 From: Kiran Kella Date: Mon, 23 Sep 2019 06:17:40 -0700 Subject: [PATCH 2/7] Add redis-tools dependencies in the docker nat compilation --- dockers/docker-nat/Dockerfile.j2 | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/dockers/docker-nat/Dockerfile.j2 b/dockers/docker-nat/Dockerfile.j2 index 75dc0e1c842d..a567db017854 100644 --- a/dockers/docker-nat/Dockerfile.j2 +++ b/dockers/docker-nat/Dockerfile.j2 @@ -1,4 +1,5 @@ -FROM {{ docker_nat_load_image }} +{% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %} +FROM docker-config-engine-stretch ARG docker_container_name RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf @@ -10,10 +11,21 @@ ENV DEBIAN_FRONTEND=noninteractive ## Install redis-tools dependencies ## TODO: implicitly install dependencies -RUN apt-get update \ - && apt-get install -f -y libdbus-1-3 libdaemon0 libjansson4 libpython2.7 \ - && apt-get -y install libjemalloc1 \ - && apt-get -y install conntrack +RUN apt-get update \ +&& apt-get install -f -y \ + libdbus-1-3 \ + libdaemon0 \ + libjansson4 \ + libpython2.7 \ + libatomic1 \ + libjemalloc1 \ + liblua5.1-0 \ + lua-bitop \ + lua-cjson \ + libelf1 \ + libmnl0 \ + bridge-utils \ + conntrack COPY \ {% for deb in docker_nat_debs.split(' ') -%} From 59565b7fc3f462e48c3df28e2d392473ce3584b7 Mon Sep 17 00:00:00 2001 From: Kiran Kella Date: Fri, 8 Nov 2019 04:49:58 -0800 Subject: [PATCH 3/7] Addressed review comments --- dockers/docker-nat/Dockerfile.j2 | 17 +++++----- dockers/docker-nat/restore_nat_entries.py | 38 ++++++++++++++--------- 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/dockers/docker-nat/Dockerfile.j2 b/dockers/docker-nat/Dockerfile.j2 index a567db017854..3cfbd99e95e1 100644 --- a/dockers/docker-nat/Dockerfile.j2 +++ b/dockers/docker-nat/Dockerfile.j2 @@ -1,4 +1,4 @@ -{% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %} +{% from "dockers/dockerfile-macros.j2" import install_debian_packages, copy_files %} FROM docker-config-engine-stretch ARG docker_container_name @@ -27,16 +27,13 @@ RUN apt-get update \ bridge-utils \ conntrack -COPY \ -{% for deb in docker_nat_debs.split(' ') -%} -debs/{{ deb }}{{' '}} -{%- endfor -%} -debs/ +{% if docker_nat_debs.strip() -%} +# Copy locally-built Debian package dependencies +{{copy_files ("debs/", docker_nat_debs.split(' '), "/debs/") }} -RUN dpkg -i \ -{% for deb in docker_nat_debs.split(' ') -%} -debs/{{ deb }}{{' '}} -{%- endfor %} +# Install locally-built Debian packages and implicitly install their dependencies +{{ install_debian_packages(docker_nat_debs.split(' ')) }} +{%- endif %} COPY ["start.sh", "/usr/bin/"] COPY ["supervisord.conf", "/etc/supervisor/conf.d/"] diff --git a/dockers/docker-nat/restore_nat_entries.py b/dockers/docker-nat/restore_nat_entries.py index a6488495f0da..ea56f113eaab 100755 --- a/dockers/docker-nat/restore_nat_entries.py +++ b/dockers/docker-nat/restore_nat_entries.py @@ -16,24 +16,31 @@ import swsssdk from swsscommon import swsscommon import logging +import logging.handlers import re import os -logging.getLogger("scapy.runtime").setLevel(logging.ERROR) +WARM_BOOT_FILE_DIR = '/var/warmboot/nat/' +NAT_WARM_BOOT_FILE = 'nat_entries.dump' +IP_PROTO_TCP = '6' + +MATCH_CONNTRACK_ENTRY = '^(\w+)\s+(\d+).*src=([\d.]+)\s+dst=([\d.]+)\s+sport=(\d+)\s+dport=(\d+).*src=([\d.]+)\s+dst=([\d.]+)\s+sport=(\d+)\s+dport=(\d+)' logger = logging.getLogger(__name__) -logger.setLevel(logging.WARNING) -logger.addHandler(logging.NullHandler()) +logger.setLevel(logging.INFO) +handler = logging.handlers.SysLogHandler(address = '/dev/log') +logger.addHandler(handler) def add_nat_conntrack_entry_in_kernel(ipproto, srcip, dstip, srcport, dstport, natsrcip, natdstip, natsrcport, natdstport): # pyroute2 doesn't have support for adding conntrack entries via netlink yet. So, invoking the conntrack utility to add the entries. state = '' - if (ipproto == '6'): + if (ipproto == IP_PROTO_TCP): state = ' --state ESTABLISHED ' ctcmd = 'conntrack -I -n ' + natdstip + ':' + natdstport + ' -g ' + natsrcip + ':' + natsrcport + \ ' --protonum ' + ipproto + state + ' --timeout 600 --src ' + srcip + ' --sport ' + srcport + \ ' --dst ' + dstip + ' --dport ' + dstport + ' -u ASSURED' subprocess.call(ctcmd, shell=True) + logger.info("Restored NAT entry: {}".format(ctcmd)) # Set the statedb "NAT_RESTORE_TABLE|Flags", so natsyncd can start reconciliation def set_statedb_nat_restore_done(): @@ -46,19 +53,20 @@ def set_statedb_nat_restore_done(): # This function is to restore the kernel nat entries based on the saved nat entries. def restore_update_kernel_nat_entries(filename): # Read the entries from nat_entries.dump file and add them to kernel + conntrack_match_pattern = re.compile(r'{}'.format(MATCH_CONNTRACK_ENTRY)) with open(filename, 'r') as fp: for line in fp: - ctline = re.findall(r'^(\w+)\s+(\d+).*src=([\d.]+)\s+dst=([\d.]+)\s+sport=(\d+)\s+dport=(\d+).*src=([\d.]+)\s+dst=([\d.]+)\s+sport=(\d+)\s+dport=(\d+)', line) + ctline = conntrack_match_pattern.findall(line) if not ctline: continue - cmdargs = ctline[0] - if ((cmdargs[0] != 'tcp') and (cmdargs[0] != 'udp')): + cmdargs = list(ctline.pop(0)) + proto = cmdargs.pop(0) + if proto not in ('tcp', 'udp'): continue - add_nat_conntrack_entry_in_kernel(cmdargs[1], cmdargs[2], cmdargs[3], cmdargs[4], cmdargs[5], cmdargs[6], cmdargs[7], cmdargs[8], cmdargs[9]) + add_nat_conntrack_entry_in_kernel(*cmdargs) def main(): - - print("restore_nat_entries service is started") + logger.info("restore_nat_entries service is started") # Use warmstart python binding to check warmstart information warmstart = swsscommon.WarmStart() @@ -67,29 +75,29 @@ def main(): # if swss or system warm reboot not enabled, don't run if not warmstart.isWarmStart(): - print("restore_nat_entries service is skipped as warm restart not enabled") + logger.info("restore_nat_entries service is skipped as warm restart not enabled") return # NAT restart not system warm reboot, set statedb directly if not warmstart.isSystemWarmRebootEnabled(): set_statedb_nat_restore_done() - print("restore_nat_entries service is done as system warm reboot not enabled") + logger.info("restore_nat_entries service is done as system warm reboot not enabled") return # Program the nat conntrack entries in the kernel by reading the # entries from nat_entries.dump try: - restore_update_kernel_nat_entries('/var/warmboot/nat/nat_entries.dump') + restore_update_kernel_nat_entries(WARM_BOOT_FILE_DIR + NAT_WARM_BOOT_FILE) except Exception as e: logger.exception(str(e)) sys.exit(1) # Remove the dump file after restoration - os.remove('/var/warmboot/nat/nat_entries.dump') + os.remove(WARM_BOOT_FILE_DIR + NAT_WARM_BOOT_FILE) # set statedb to signal other processes like natsyncd set_statedb_nat_restore_done() - print("restore_nat_entries service is done for system warmreboot") + logger.info("restore_nat_entries service is done for system warmreboot") return if __name__ == '__main__': From f5cfe6b1bca2c4d134e76a560a22b8bb444004b2 Mon Sep 17 00:00:00 2001 From: Kiran Kella Date: Mon, 11 Nov 2019 04:26:16 -0800 Subject: [PATCH 4/7] add natsyncd to warm-boot finalizer list --- files/image_config/warmboot-finalizer/finalize-warmboot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/image_config/warmboot-finalizer/finalize-warmboot.sh b/files/image_config/warmboot-finalizer/finalize-warmboot.sh index 32c9c8444cc3..ba473feab0ab 100755 --- a/files/image_config/warmboot-finalizer/finalize-warmboot.sh +++ b/files/image_config/warmboot-finalizer/finalize-warmboot.sh @@ -3,7 +3,7 @@ VERBOSE=no # Check components -COMP_LIST="orchagent neighsyncd bgp" +COMP_LIST="orchagent neighsyncd bgp natsyncd" EXP_STATE="reconciled" ASSISTANT_SCRIPT="/usr/bin/neighbor_advertiser" From c6192b37aa4c825c76d0e114320b5bac354ee2ef Mon Sep 17 00:00:00 2001 From: Kiran Kella Date: Mon, 11 Nov 2019 04:57:44 -0800 Subject: [PATCH 5/7] addressed review comments --- dockers/docker-nat/start.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/dockers/docker-nat/start.sh b/dockers/docker-nat/start.sh index 0d89d990d29e..e1f303fee6f2 100755 --- a/dockers/docker-nat/start.sh +++ b/dockers/docker-nat/start.sh @@ -9,8 +9,6 @@ supervisorctl start rsyslogd supervisorctl start natmgrd -sleep 5 - supervisorctl start natsyncd supervisorctl start restore_nat_entries From 423e20d9c1ee06d90d42d0648e76326af67e14f0 Mon Sep 17 00:00:00 2001 From: Kiran Kella Date: Tue, 12 Nov 2019 03:47:19 -0800 Subject: [PATCH 6/7] using swsscommon.DBConnector instead of swsssdk.SonicV2Connector --- dockers/docker-nat/restore_nat_entries.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dockers/docker-nat/restore_nat_entries.py b/dockers/docker-nat/restore_nat_entries.py index ea56f113eaab..9fb62e82f573 100755 --- a/dockers/docker-nat/restore_nat_entries.py +++ b/dockers/docker-nat/restore_nat_entries.py @@ -13,7 +13,6 @@ import sys import subprocess -import swsssdk from swsscommon import swsscommon import logging import logging.handlers @@ -25,6 +24,7 @@ IP_PROTO_TCP = '6' MATCH_CONNTRACK_ENTRY = '^(\w+)\s+(\d+).*src=([\d.]+)\s+dst=([\d.]+)\s+sport=(\d+)\s+dport=(\d+).*src=([\d.]+)\s+dst=([\d.]+)\s+sport=(\d+)\s+dport=(\d+)' +REDIS_SOCK = "/var/run/redis/redis.sock" logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -44,10 +44,10 @@ def add_nat_conntrack_entry_in_kernel(ipproto, srcip, dstip, srcport, dstport, n # Set the statedb "NAT_RESTORE_TABLE|Flags", so natsyncd can start reconciliation def set_statedb_nat_restore_done(): - db = swsssdk.SonicV2Connector(host='127.0.0.1') - db.connect(db.STATE_DB, False) - db.set(db.STATE_DB, 'NAT_RESTORE_TABLE|Flags', 'restored', 'true') - db.close(db.STATE_DB) + statedb = swsscommon.DBConnector(swsscommon.STATE_DB, REDIS_SOCK, 0) + tbl = swsscommon.Table(statedb, "NAT_RESTORE_TABLE") + fvs = swsscommon.FieldValuePairs([("restored", "true")]) + tbl.set("Flags", fvs) return # This function is to restore the kernel nat entries based on the saved nat entries. From 031d8d6a36aefdb6a3e30d2971587c2cabffe0f8 Mon Sep 17 00:00:00 2001 From: Kiran Kella Date: Tue, 12 Nov 2019 20:51:31 -0800 Subject: [PATCH 7/7] Enable NAT application in docker-sonic-vs --- platform/vs/docker-sonic-vs/start.sh | 4 ++++ platform/vs/docker-sonic-vs/supervisord.conf | 16 ++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/platform/vs/docker-sonic-vs/start.sh b/platform/vs/docker-sonic-vs/start.sh index 614541961c87..dd9fee4deb3d 100755 --- a/platform/vs/docker-sonic-vs/start.sh +++ b/platform/vs/docker-sonic-vs/start.sh @@ -70,6 +70,10 @@ supervisorctl start vxlanmgrd supervisorctl start sflowmgrd +supervisorctl start natmgrd + +supervisorctl start natsyncd + # Start arp_update when VLAN exists VLAN=`sonic-cfggen -d -v 'VLAN.keys() | join(" ") if VLAN'` if [ "$VLAN" != "" ]; then diff --git a/platform/vs/docker-sonic-vs/supervisord.conf b/platform/vs/docker-sonic-vs/supervisord.conf index 143fe49d44a9..3a7acfd20bbe 100644 --- a/platform/vs/docker-sonic-vs/supervisord.conf +++ b/platform/vs/docker-sonic-vs/supervisord.conf @@ -188,3 +188,19 @@ autostart=false autorestart=false stdout_logfile=syslog stderr_logfile=syslog + +[program:natmgrd] +command=/usr/bin/natmgrd +priority=23 +autostart=false +autorestart=false +stdout_logfile=syslog +stderr_logfile=syslog + +[program:natsyncd] +command=/usr/bin/natsyncd +priority=24 +autostart=false +autorestart=false +stdout_logfile=syslog +stderr_logfile=syslog