Build infrastructure changes to support sflow docker and utilities

dockers/docker-sflow/Dockerfile.j2

@@ -0,0 +1,33 @@
+{% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
+FROM docker-config-engine-stretch
+
+ARG docker_container_name
+RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf
+
+## Make apt-get non-interactive
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update      && \
+    apt-get install -f -y  \
+            dmidecode      \
+            libmnl0=1.0.4-2
+
+{% if docker_sflow_debs.strip() -%}
+# Copy locally-built Debian package dependencies
+{{ copy_files("debs/", docker_sflow_debs.split(' '), "/debs/") }}
+
+# Install locally-built Debian packages and implicitly install their dependencies
+{{ install_debian_packages(docker_sflow_debs.split(' ')) }}
+{%- endif %}
+
+RUN rm -rf /etc/dbus-1
+
+RUN apt-get clean -y      && \
+    apt-get autoclean -y  && \
+    apt-get autoremove -y && \
+    rm -rf /debs
+
+COPY ["start.sh", "/usr/bin/"]
+COPY ["supervisord.conf", "/etc/supervisor/conf.d/"]
+
+ENTRYPOINT ["/usr/bin/supervisord"]

dockers/docker-sflow/base_image_files/psample

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+DOCKER_EXEC_FLAGS="i"
+
+# Determine whether stdout is on a terminal
+if [ -t 1 ] ; then
+    DOCKER_EXEC_FLAGS+="t"
+fi
+
+docker exec -$DOCKER_EXEC_FLAGS sflow psample "$@"

dockers/docker-sflow/base_image_files/sflowtool

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+DOCKER_EXEC_FLAGS="i"
+
+# Determine whether stdout is on a terminal
+if [ -t 1 ] ; then
+    DOCKER_EXEC_FLAGS+="t"
+fi
+
+docker exec -$DOCKER_EXEC_FLAGS sflow sflowtool "$@"

dockers/docker-sflow/start.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+rm -f /var/run/rsyslogd.pid
+
+supervisorctl start rsyslogd
+
+supervisorctl start sflowmgrd
+
+service hsflowd restart

dockers/docker-sflow/supervisord.conf

@@ -0,0 +1,28 @@
+[supervisord]
+logfile_maxbytes=1MB
+logfile_backups=2
+nodaemon=true
+
+[program:start.sh]
+command=/usr/bin/start.sh
+priority=1
+autostart=true
+autorestart=false
+stdout_logfile=syslog
+stderr_logfile=syslog
+
+[program:rsyslogd]
+command=/usr/sbin/rsyslogd -n
+priority=2
+autostart=false
+autorestart=false
+stdout_logfile=syslog
+stderr_logfile=syslog
+
+[program:sflowmgrd]
+command=/usr/bin/sflowmgrd
+priority=3
+autostart=false
+autorestart=false
+stdout_logfile=syslog
+stderr_logfile=syslog

files/build_templates/sflow.service.j2

@@ -0,0 +1,14 @@
+[Unit]
+Description=sFlow container
+Requires=swss.service
+After=swss.service
+Before=ntp-config.service
+
+[Service]
+User={{ sonicadmin_user }}
+ExecStartPre=/usr/bin/{{docker_container_name}}.sh start
+ExecStart=/usr/bin/{{docker_container_name}}.sh wait
+ExecStop=/usr/bin/{{docker_container_name}}.sh stop
+
+[Install]
+WantedBy=multi-user.target

files/build_templates/sonic_debian_extension.j2

@@ -300,6 +300,14 @@ sudo cp $IMAGE_CONFIGS/platform/rc.local $FILESYSTEM_ROOT/etc/
 ## copy blacklist file
 sudo cp $IMAGE_CONFIGS/platform/linux_kernel_bde.conf $FILESYSTEM_ROOT/etc/modprobe.d/
 
+# Enable psample drivers to support sFlow on vs
+{% if sonic_asic_platform == "vs" %}
+sudo tee -a $FILESYSTEM_ROOT/etc/modules-load.d/modules.conf > /dev/null <<EOF
+psample
+act_sample
+EOF
+{% endif %}
+
 ## Bind docker path
 if [[ $CONFIGURED_ARCH == armhf || $CONFIGURED_ARCH == arm64 ]]; then
     sudo mkdir -p $FILESYSTEM_ROOT/dockerfs

platform/vs/docker-syncd-vs.mk

@@ -12,3 +12,5 @@ $(DOCKER_SYNCD_BASE)_DBG_DEPENDS += $(SYNCD_VS_DBG) \
                                 $(LIBSAIVS_DBG)
 
 $(DOCKER_SYNCD_BASE)_RUN_OPT += -v /host/warmboot:/var/warmboot
+
+$(DOCKER_SYNCD_BASE)_BASE_IMAGE_FILES += tc-syncd:/usr/bin/tc-syncd

platform/vs/docker-syncd-vs/Dockerfile.j2

@@ -8,6 +8,8 @@ ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update
 
+RUN apt-get install -f -y iproute2=4.20.0-2~bpo9+1 libcap2-bin=1:2.25-1
+
 COPY \
 {% for deb in docker_syncd_vs_debs.split(' ') -%}
 debs/{{ deb }}{{' '}}

platform/vs/docker-syncd-vs/base_image_files/tc-syncd

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+DOCKER_EXEC_FLAGS="i"
+
+# Determine whether stdout is on a terminal
+if [ -t 1 ] ; then
+    DOCKER_EXEC_FLAGS+="t"
+fi
+
+docker exec -$DOCKER_EXEC_FLAGS syncd tc "$@"

rules/config

@@ -95,3 +95,6 @@ FRR_USER_GID = 300
 
 # Default VS build memory preparation
 DEFAULT_VS_PREPARE_MEM = yes
+
+# ENABLE_SYSTEM_SFLOW - build docker-sonic-sflow for sFlow support
+ENABLE_SFLOW = n

rules/docker-sflow.mk

@@ -0,0 +1,35 @@
+# docker image for sFlow agent
+
+DOCKER_SFLOW_STEM = docker-sflow
+DOCKER_SFLOW = $(DOCKER_SFLOW_STEM).gz
+DOCKER_SFLOW_DBG = $(DOCKER_SFLOW_STEM)-$(DBG_IMAGE_MARK).gz
+
+$(DOCKER_SFLOW)_PATH = $(DOCKERS_PATH)/$(DOCKER_SFLOW_STEM)
+
+$(DOCKER_SFLOW)_DEPENDS += $(SWSS) $(REDIS_TOOLS) $(HSFLOWD) $(SFLOWTOOL) $(PSAMPLE)
+$(DOCKER_SFLOW)_DBG_DEPENDS = $($(DOCKER_CONFIG_ENGINE_STRETCH)_DBG_DEPENDS)
+$(DOCKER_TEAMD)_DBG_DEPENDS += $(SWSS_DBG) $(LIBSWSSCOMMON_DBG)
+$(DOCKER_SFLOW)_DBG_IMAGE_PACKAGES = $($(DOCKER_CONFIG_ENGINE_STRETCH)_DBG_IMAGE_PACKAGES)
+
+$(DOCKER_SFLOW)_LOAD_DOCKERS += $(DOCKER_CONFIG_ENGINE_STRETCH)
+
+SONIC_DOCKER_IMAGES += $(DOCKER_SFLOW)
+ifeq ($(ENABLE_SFLOW), y)
+SONIC_INSTALL_DOCKER_IMAGES += $(DOCKER_SFLOW)
+SONIC_STRETCH_DOCKERS += $(DOCKER_SFLOW)
+endif
+
+SONIC_DOCKER_DBG_IMAGES += $(DOCKER_SFLOW_DBG)
+ifeq ($(ENABLE_SFLOW), y)
+SONIC_INSTALL_DOCKER_DBG_IMAGES += $(DOCKER_SFLOW_DBG)
+SONIC_STRETCH_DBG_DOCKERS += $(DOCKER_SFLOW_DBG)
+endif
+
+$(DOCKER_SFLOW)_CONTAINER_NAME = sflow
+$(DOCKER_SFLOW)_RUN_OPT += --net=host --privileged -t
+$(DOCKER_SFLOW)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro
+$(DOCKER_SFLOW)_RUN_OPT += -v /host/warmboot:/var/warmboot
+
+$(DOCKER_SFLOW)_BASE_IMAGE_FILES += psample:/usr/bin/psample
+$(DOCKER_SFLOW)_BASE_IMAGE_FILES += sflowtool:/usr/bin/sflowtool
+

rules/sflow.mk

@@ -0,0 +1,45 @@
+# host-sflow package
+
+HSFLOWD_VERSION = 2.0.21
+HSFLOWD_SUBVERSION = 4
+export HSFLOWD_VERSION HSFLOWD_SUBVERSION
+
+HSFLOWD = hsflowd_$(HSFLOWD_VERSION)-$(HSFLOWD_SUBVERSION)_amd64.deb
+$(HSFLOWD)_DEPENDS += $(LIBHIREDIS_DEV) $(LINUX_HEADERS) $(LINUX_HEADERS_COMMON)
+$(HSFLOWD)_SRC_PATH = $(SRC_PATH)/sflow/hsflowd
+
+SONIC_MAKE_DEBS += $(HSFLOWD)
+SONIC_STRETCH_DEBS += $(HSFLOWD)
+export HSFLOWD
+
+# sflowtool package
+
+SFLOWTOOL_VERSION = 5.04
+export SFLOWTOOL_VERSION
+
+SFLOWTOOL = sflowtool_$(SFLOWTOOL_VERSION)_amd64.deb
+$(SFLOWTOOL)_SRC_PATH = $(SRC_PATH)/sflow/sflowtool
+
+SONIC_MAKE_DEBS += $(SFLOWTOOL)
+SONIC_STRETCH_DEBS += $(SFLOWTOOL)
+export SFLOWTOOL
+
+# libmnl & psample packages
+
+LIBMNL_DEV_VERSION = 1.0.4-2
+export LIBMNL_DEV_VERSION
+LIBMNL_DEV = libmnl-dev_$(LIBMNL_DEV_VERSION)_amd64.deb
+$(LIBMNL_DEV)_URL = http://ftp.debian.org/debian/pool/main/libm/libmnl/libmnl-dev_1.0.4-2_amd64.deb
+SONIC_ONLINE_DEBS += $(LIBMNL_DEV)
+
+PSAMPLE_VERSION = 1.1
+PSAMPLE_SUBVERSION = 1
+export PSAMPLE_VERSION PSAMPLE_SUBVERSION
+
+PSAMPLE = psample_$(PSAMPLE_VERSION)-$(PSAMPLE_SUBVERSION)_amd64.deb
+$(PSAMPLE)_DEPENDS += $(LIBMNL_DEV)
+$(PSAMPLE)_SRC_PATH = $(SRC_PATH)/sflow/psample
+
+SONIC_MAKE_DEBS += $(PSAMPLE)
+SONIC_STRETCH_DEBS += $(PSAMPLE)
+export PSAMPLE

slave.mk

@@ -99,6 +99,10 @@ ifeq ($(SONIC_INSTALL_DEBUG_TOOLS),y)
 INSTALL_DEBUG_TOOLS = y
 endif
 
+ifeq ($(SONIC_ENABLE_SFLOW),y)
+ENABLE_SFLOW = y
+endif
+
 include $(RULES_PATH)/functions
 include $(RULES_PATH)/*.mk
 ifneq ($(CONFIGURED_PLATFORM), undefined)
@@ -183,6 +187,7 @@ $(info "KERNEL_PROCURE_METHOD"           : "$(KERNEL_PROCURE_METHOD)")
 $(info "BUILD_TIMESTAMP"                 : "$(BUILD_TIMESTAMP)")
 $(info "BLDENV"                          : "$(BLDENV)")
 $(info "VS_PREPARE_MEM"                  : "$(VS_PREPARE_MEM)")
+$(info "ENABLE_SFLOW"                    : "$(ENABLE_SFLOW)")
 $(info )
 
 ifeq ($(SONIC_USE_DOCKER_BUILDKIT),y)

src/sflow/hsflowd/Makefile

@@ -0,0 +1,21 @@
+.ONESHELL:
+SHELL = /bin/bash
+.SHELLFLAGS += -e
+
+MAIN_TARGET = $(HSFLOWD)
+
+$(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
+	rm -fr ./host-sflow
+	git clone https://github.com/sflow/host-sflow
+
+	pushd ./host-sflow
+	git checkout -b sflow -f 5f171bb
+
+        # Apply patch series
+	stg init
+	stg import -s ../patch/series
+
+	make deb FEATURES="SONIC"
+	mv $* $(DEST)/
+	popd
+

src/sflow/hsflowd/patch/0001-host_sflow_psample.patch

@@ -0,0 +1,39 @@
+diff -ruN a/src/Linux/linux/psample.h b/src/Linux/linux/psample.h
+--- a/src/Linux/linux/psample.h	1969-12-31 16:00:00.000000000 -0800
++++ b/src/Linux/linux/psample.h	2019-07-20 08:45:58.715748881 -0700
+@@ -0,0 +1,35 @@
++#ifndef __UAPI_PSAMPLE_H
++#define __UAPI_PSAMPLE_H
++
++enum {
++	/* sampled packet metadata */
++	PSAMPLE_ATTR_IIFINDEX,
++	PSAMPLE_ATTR_OIFINDEX,
++	PSAMPLE_ATTR_ORIGSIZE,
++	PSAMPLE_ATTR_SAMPLE_GROUP,
++	PSAMPLE_ATTR_GROUP_SEQ,
++	PSAMPLE_ATTR_SAMPLE_RATE,
++	PSAMPLE_ATTR_DATA,
++
++	/* commands attributes */
++	PSAMPLE_ATTR_GROUP_REFCOUNT,
++
++	__PSAMPLE_ATTR_MAX
++};
++
++enum psample_command {
++	PSAMPLE_CMD_SAMPLE,
++	PSAMPLE_CMD_GET_GROUP,
++	PSAMPLE_CMD_NEW_GROUP,
++	PSAMPLE_CMD_DEL_GROUP,
++};
++
++/* Can be overridden at runtime by module option */
++#define PSAMPLE_ATTR_MAX (__PSAMPLE_ATTR_MAX - 1)
++
++#define PSAMPLE_NL_MCGRP_CONFIG_NAME "config"
++#define PSAMPLE_NL_MCGRP_SAMPLE_NAME "packets"
++#define PSAMPLE_GENL_NAME "psample"
++#define PSAMPLE_GENL_VERSION 1
++
++#endif

src/sflow/hsflowd/patch/series

@@ -0,0 +1 @@
+0001-host_sflow_psample.patch

src/sflow/psample/Makefile

@@ -0,0 +1,19 @@
+.ONESHELL:
+SHELL = /bin/bash
+.SHELLFLAGS += -e
+
+MAIN_TARGET = $(PSAMPLE)
+
+$(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
+
+	rm -fr ./libpsample
+	git clone https://github.com/Mellanox/libpsample.git
+	cp -r debian libpsample
+
+	pushd ./libpsample
+	git checkout -b libpsample -f e48fad2
+	dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS)
+	popd
+
+	mv $* $(DEST)/
+

src/sflow/psample/debian/changelog

@@ -0,0 +1,6 @@
+psample (1.1-1) UNRELEASED; urgency=medium
+
+  [ DellEMC ]
+  * Initial release : based on https://github.com/Mellanox/libpsample
+
+ -- DellEMC <[email protected]>  Mon, 29 Jul 2019 07:08:02 -0400

src/sflow/psample/debian/compat

@@ -0,0 +1 @@
+9

src/sflow/psample/debian/control

@@ -0,0 +1,11 @@
+Source: psample
+Maintainer: DellEMC <[email protected]>
+Section: devel
+Priority: optional
+Build-Depends: dh-exec (>=0.3), debhelper (>= 9), autotools-dev
+Standards-Version: 1.0.0
+
+Package: psample
+Architecture: any
+Depends: ${shlibs:Depends}
+Description: This package contains psample from https://github.com/Mellanox/libpsample

src/sflow/psample/debian/psample.install

@@ -0,0 +1,4 @@
+bin/psample usr/bin
+lib/libpsample.so.1.0 lib/x86_64-linux-gnu
+lib/libpsample.so.1 lib/x86_64-linux-gnu
+lib/libpsample.so lib/x86_64-linux-gnu

src/sflow/psample/debian/rules

@@ -0,0 +1,19 @@
+#!/usr/bin/make -f
+
+# main packaging script based on dh7 syntax
+%:
+	dh $@
+
+override_dh_auto_build:
+	cmake . && make
+
+override_dh_auto_install:
+
+override_dh_auto_test:
+
+override_dh_auto_clean:
+
+override_dh_strip:
+
+override_dh_clean:
+