From 099f693aac2904b927f52f8afd873e0009ee2f60 Mon Sep 17 00:00:00 2001 From: Tamer Ahmed Date: Mon, 6 Jul 2020 09:59:45 -0700 Subject: [PATCH 1/5] [telemetry] Call sonic-cfggen Once sonic-cfggen call is slow and this is taking place in the SONiC boot up process. The change uses templates to assemble all required vars into single template file. With this change, telemetry now callse once into sonic-cfggen. signed-off-by: Tamer Ahmed --- dockers/docker-base-buster/Dockerfile.j2 | 4 ++- dockers/docker-base-stretch/Dockerfile.j2 | 4 ++- dockers/docker-sonic-telemetry/Dockerfile.j2 | 2 +- dockers/docker-sonic-telemetry/telemetry.sh | 30 ++++++++++--------- .../docker-sonic-telemetry/telemetry_vars.j2 | 5 ++++ 5 files changed, 28 insertions(+), 17 deletions(-) create mode 100644 dockers/docker-sonic-telemetry/telemetry_vars.j2 diff --git a/dockers/docker-base-buster/Dockerfile.j2 b/dockers/docker-base-buster/Dockerfile.j2 index 1959bb56cb7b..2d7bd887d2e8 100644 --- a/dockers/docker-base-buster/Dockerfile.j2 +++ b/dockers/docker-base-buster/Dockerfile.j2 @@ -64,7 +64,9 @@ RUN apt-get update && \ net-tools \ # for arm arch: Installing j2cli dependency package MarkupSafe from source relies on weeksetuptools and wheel python-setuptools \ - python-wheel + python-wheel \ +# for processing/handling json files in bash environment + jq # For templating RUN pip install j2cli diff --git a/dockers/docker-base-stretch/Dockerfile.j2 b/dockers/docker-base-stretch/Dockerfile.j2 index f573e94b78d7..f563aee8d9ea 100644 --- a/dockers/docker-base-stretch/Dockerfile.j2 +++ b/dockers/docker-base-stretch/Dockerfile.j2 @@ -62,7 +62,9 @@ RUN apt-get update && \ net-tools \ # for arm arch: Installing j2cli dependency package MarkupSafe from source relies on weeksetuptools and wheel python-setuptools \ - python-wheel + python-wheel \ +# for processing json files in bash environment + jq # For templating RUN pip install j2cli diff --git a/dockers/docker-sonic-telemetry/Dockerfile.j2 b/dockers/docker-sonic-telemetry/Dockerfile.j2 index d3b95c3b4002..b878266f1c6c 100644 --- a/dockers/docker-sonic-telemetry/Dockerfile.j2 +++ b/dockers/docker-sonic-telemetry/Dockerfile.j2 @@ -22,7 +22,7 @@ RUN apt-get clean -y && \ apt-get autoremove -y && \ rm -rf /debs -COPY ["start.sh", "telemetry.sh", "dialout.sh", "/usr/bin/"] +COPY ["start.sh", "telemetry.sh", "dialout.sh", "telemetry_vars.j2", "/usr/bin/"] COPY ["supervisord.conf", "/etc/supervisor/conf.d/"] COPY ["files/supervisor-proc-exit-listener", "/usr/bin"] COPY ["critical_processes", "/etc/supervisor"] diff --git a/dockers/docker-sonic-telemetry/telemetry.sh b/dockers/docker-sonic-telemetry/telemetry.sh index b8f7fffb3ebb..bfef915c60be 100755 --- a/dockers/docker-sonic-telemetry/telemetry.sh +++ b/dockers/docker-sonic-telemetry/telemetry.sh @@ -2,36 +2,38 @@ # Try to read telemetry and certs config from ConfigDB. # Use default value if no valid config exists -X509=`sonic-cfggen -d -v "DEVICE_METADATA['x509']"` -gnmi=`sonic-cfggen -d -v "TELEMETRY['gnmi']"` -certs=`sonic-cfggen -d -v "TELEMETRY['certs']"` +TELEMETRY_VARS=`sonic-cfggen -d -t telemetry_vars.j2` +TELEMETRY_VARS=${TELEMETRY_VARS//[\']/\"} +X509=$(echo $TELEMETRY_VARS | jq -r '.x509') +GNMI=$(echo $TELEMETRY_VARS | jq -r '.gnmi') +CERTS=$(echo $TELEMETRY_VARS | jq -r '.certs') TELEMETRY_ARGS=" -logtostderr" export CVL_SCHEMA_PATH=/usr/sbin/schema -if [ -n "$certs" ]; then - SERVER_CRT=`sonic-cfggen -d -v "TELEMETRY['certs']['server_crt']"` - SERVER_KEY=`sonic-cfggen -d -v "TELEMETRY['certs']['server_key']"` +if [ -n "$CERTS" ]; then + SERVER_CRT=$(echo $CERTS | jq -r '.server_crt') + SERVER_KEY=$(echo $CERTS | jq -r '.server_key') if [ -z $SERVER_CRT ] || [ -z $SERVER_KEY ]; then TELEMETRY_ARGS+=" --insecure" else TELEMETRY_ARGS+=" --server_crt $SERVER_CRT --server_key $SERVER_KEY " fi - CA_CRT=`sonic-cfggen -d -v "TELEMETRY['certs']['ca_crt']"` + CA_CRT=$(echo $CERTS | jq -r '.ca_crt') if [ ! -z $CA_CRT ]; then TELEMETRY_ARGS+=" --ca_crt $CA_CRT" fi elif [ -n "$X509" ]; then - SERVER_CRT=`sonic-cfggen -d -v "DEVICE_METADATA['x509']['server_crt']"` - SERVER_KEY=`sonic-cfggen -d -v "DEVICE_METADATA['x509']['server_key']"` + SERVER_CRT=$(echo $X509 | jq -r '.server_crt') + SERVER_KEY=$(echo $X509 | jq -r '.server_key') if [ -z $SERVER_CRT ] || [ -z $SERVER_KEY ]; then TELEMETRY_ARGS+=" --insecure" else TELEMETRY_ARGS+=" --server_crt $SERVER_CRT --server_key $SERVER_KEY " fi - CA_CRT=`sonic-cfggen -d -v "DEVICE_METADATA['x509']['ca_crt']"` + CA_CRT=$(echo $X509 | jq -r '.ca_crt') if [ ! -z $CA_CRT ]; then TELEMETRY_ARGS+=" --ca_crt $CA_CRT" fi @@ -40,19 +42,19 @@ else fi # If no configuration entry exists for TELEMETRY, create one default port -if [ -z "$gnmi" ]; then +if [ -z "$GNMI" ]; then sonic-db-cli CONFIG_DB hset "TELEMETRY|gnmi" port 8080 fi -PORT=`sonic-cfggen -d -v "TELEMETRY['gnmi']['port']"` +PORT=$(echo $GNMI | jq -r '.port') TELEMETRY_ARGS+=" --port $PORT" -CLIENT_AUTH=`sonic-cfggen -d -v "TELEMETRY['gnmi']['client_auth']"` +CLIENT_AUTH=$(echo $GNMI | jq -r '.client_auth') if [ -z $CLIENT_AUTH ] || [ $CLIENT_AUTH == "false" ]; then TELEMETRY_ARGS+=" --allow_no_client_auth" fi -LOG_LEVEL=`sonic-cfggen -d -v "TELEMETRY['gnmi']['log_level']"` +LOG_LEVEL=$(echo $GNMI | jq -r '.log_level') if [ ! -z $LOG_LEVEL ]; then TELEMETRY_ARGS+=" -v=$LOG_LEVEL" else diff --git a/dockers/docker-sonic-telemetry/telemetry_vars.j2 b/dockers/docker-sonic-telemetry/telemetry_vars.j2 new file mode 100644 index 000000000000..9ba523e61bff --- /dev/null +++ b/dockers/docker-sonic-telemetry/telemetry_vars.j2 @@ -0,0 +1,5 @@ +{ + "x509" : "{% if DEVICE_METADATA['x509'] is defined %}{{ DEVICE_METADATA['x509'] }}{% endif %}", + "gnmi" : "{% if DEVICE_METADATA['gnmi'] is defined %}{{ DEVICE_METADATA['gnmi'] }}{% endif %}", + "certs": "{% if DEVICE_METADATA['certs'] is defined %}{{ DEVICE_METADATA['certs'] }}{% endif %}" +} From 84359277930e8a06d41b77e908a7f7ff5915c5c8 Mon Sep 17 00:00:00 2001 From: Tamer Ahmed Date: Mon, 6 Jul 2020 15:25:39 -0700 Subject: [PATCH 2/5] review comments --- dockers/docker-sonic-telemetry/telemetry.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockers/docker-sonic-telemetry/telemetry.sh b/dockers/docker-sonic-telemetry/telemetry.sh index bfef915c60be..be4c7dbe5109 100755 --- a/dockers/docker-sonic-telemetry/telemetry.sh +++ b/dockers/docker-sonic-telemetry/telemetry.sh @@ -2,7 +2,7 @@ # Try to read telemetry and certs config from ConfigDB. # Use default value if no valid config exists -TELEMETRY_VARS=`sonic-cfggen -d -t telemetry_vars.j2` +TELEMETRY_VARS=$(sonic-cfggen -d -t telemetry_vars.j2) TELEMETRY_VARS=${TELEMETRY_VARS//[\']/\"} X509=$(echo $TELEMETRY_VARS | jq -r '.x509') GNMI=$(echo $TELEMETRY_VARS | jq -r '.gnmi') From 970c4452fefe45320aaff81e24cdf926bd3d2c0b Mon Sep 17 00:00:00 2001 From: Tamer Ahmed Date: Mon, 6 Jul 2020 16:14:34 -0700 Subject: [PATCH 3/5] Use self assigned port in case gnmi does not define any. --- dockers/docker-sonic-telemetry/telemetry.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dockers/docker-sonic-telemetry/telemetry.sh b/dockers/docker-sonic-telemetry/telemetry.sh index be4c7dbe5109..18356e0945e2 100755 --- a/dockers/docker-sonic-telemetry/telemetry.sh +++ b/dockers/docker-sonic-telemetry/telemetry.sh @@ -43,10 +43,11 @@ fi # If no configuration entry exists for TELEMETRY, create one default port if [ -z "$GNMI" ]; then - sonic-db-cli CONFIG_DB hset "TELEMETRY|gnmi" port 8080 + PORT=8080 + sonic-db-cli CONFIG_DB hset "TELEMETRY|gnmi" port $PORT +else + PORT=$(echo $GNMI | jq -r '.port') fi - -PORT=$(echo $GNMI | jq -r '.port') TELEMETRY_ARGS+=" --port $PORT" CLIENT_AUTH=$(echo $GNMI | jq -r '.client_auth') From ed26ecb4ce317e2e48b8750afc501880015cb4f6 Mon Sep 17 00:00:00 2001 From: Tamer Ahmed Date: Tue, 7 Jul 2020 10:05:20 -0700 Subject: [PATCH 4/5] use TELEMETRY with gnmi and certs and DEVICE_METADATA with x509 --- dockers/docker-sonic-telemetry/telemetry_vars.j2 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dockers/docker-sonic-telemetry/telemetry_vars.j2 b/dockers/docker-sonic-telemetry/telemetry_vars.j2 index 9ba523e61bff..ed5d07690372 100644 --- a/dockers/docker-sonic-telemetry/telemetry_vars.j2 +++ b/dockers/docker-sonic-telemetry/telemetry_vars.j2 @@ -1,5 +1,5 @@ { - "x509" : "{% if DEVICE_METADATA['x509'] is defined %}{{ DEVICE_METADATA['x509'] }}{% endif %}", - "gnmi" : "{% if DEVICE_METADATA['gnmi'] is defined %}{{ DEVICE_METADATA['gnmi'] }}{% endif %}", - "certs": "{% if DEVICE_METADATA['certs'] is defined %}{{ DEVICE_METADATA['certs'] }}{% endif %}" -} + "certs": "{% if "certs" in TELEMETRY.keys() %}{{ TELEMETRY["certs"] }}{% endif %}", + "gnmi" : "{% if "gnmi" in TELEMETRY.keys() %}{{ TELEMETRY["gnmi"] }}{% endif %}", + "x509" : "{% if "x509" in DEVICE_METADATA.keys() %}{{ DEVICE_METADATA["x509"] }}{% endif %}" +} \ No newline at end of file From 0723fee0ef4672bff9fe11b33f7598da9d50b04e Mon Sep 17 00:00:00 2001 From: Tamer Ahmed Date: Tue, 7 Jul 2020 14:55:06 -0700 Subject: [PATCH 5/5] add new line at end of file --- dockers/docker-sonic-telemetry/telemetry_vars.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockers/docker-sonic-telemetry/telemetry_vars.j2 b/dockers/docker-sonic-telemetry/telemetry_vars.j2 index ed5d07690372..687781e1b9b2 100644 --- a/dockers/docker-sonic-telemetry/telemetry_vars.j2 +++ b/dockers/docker-sonic-telemetry/telemetry_vars.j2 @@ -2,4 +2,4 @@ "certs": "{% if "certs" in TELEMETRY.keys() %}{{ TELEMETRY["certs"] }}{% endif %}", "gnmi" : "{% if "gnmi" in TELEMETRY.keys() %}{{ TELEMETRY["gnmi"] }}{% endif %}", "x509" : "{% if "x509" in DEVICE_METADATA.keys() %}{{ DEVICE_METADATA["x509"] }}{% endif %}" -} \ No newline at end of file +}