Merge Azure/master to local fork

README.md

@@ -4,9 +4,10 @@
 # Description
 Tools for managing, configuring and monitoring SONiC
 
+# CII Best Practices
+[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/3933/badge)](https://bestpractices.coreinfrastructure.org/projects/3933)
 
 # Contribution guide
-
 All contributors must sign a contribution license agreement before contributions can be accepted.  
 [How to become a contributer](https://github.com/Azure/SONiC/wiki/Becoming-a-contributor)
 

ansible/README.testbed.md

@@ -2,6 +2,7 @@
 
 - [Overview](doc/README.testbed.Overview.md)
 - [Setup](doc/README.testbed.Setup.md)
+  - [Virtual Switch Testbed Setup](doc/README.testbed.VsSetup.md)
 - [Topology](doc/README.testbed.Topology.md)
 - [Configuration](doc/README.testbed.Config.md)
 - [Minigraph](doc/README.testbed.Minigraph.md)

ansible/TestbedProcessing.py

@@ -9,14 +9,14 @@
 
 Requirement:
     python version: 2.X
-    python package: PyYAML 3.12 (or later) 
-    
+    python package: PyYAML 3.12 (or later)
+
 PyYaml Install Instructions:
-    [1] Download PyYAML from https://pyyaml.org/wiki/PyYAML 
+    [1] Download PyYAML from https://pyyaml.org/wiki/PyYAML
     [2] Unpack the archive
     [3] Install the package by executing (python setup.py install)
-    [4] Test if installation was successful (python setup.py test) 
-    
+    [4] Test if installation was successful (python setup.py test)
+
 Usage:
     put TestbedProcessing.py and testbed.yaml under sonic-mgmt/ansible
     python TestbedProcessing.py
@@ -25,12 +25,12 @@
 Arguments:
     -i : the testbed.yaml file to parse
     -basedir : the basedir for the project
-    -backupdir : the backup directory for the files  
+    -backupdir : the backup directory for the files
 
 Script Procedure
-    [1] Backup the files we will be copying 
+    [1] Backup the files we will be copying
     [2] Load testbed.yaml into dictionaries for easy processing
-    [3] Generate the files via methods defined below 
+    [3] Generate the files via methods defined below
 """
 
 # ARGUMENTS TO PARSE
@@ -86,7 +86,7 @@
 
 """
 represent_none(self, _)
-modifies yaml to replace null values with blanks 
+modifies yaml to replace null values with blanks
 SOURCE: https://stackoverflow.com/questions/37200150/can-i-dump-blank-instead-of-null-in-yaml-pyyaml/37201633#3720163
 """
 def represent_none(self, _):
@@ -98,7 +98,7 @@ def represent_none(self, _):
 generateDictionary(data, result, category)
 @:parameter data - the dictionary to iterate through
 @:parameter result - the resulting dictionary
-Generates the dictionaries that are used when creating csv, yml, or text files 
+Generates the dictionaries that are used when creating csv, yml, or text files
 """
 def generateDictionary(data, result, category):
     for key, value in data[category].items():
@@ -108,7 +108,7 @@ def generateDictionary(data, result, category):
 """
 makeMain(data, outfile)
 @:parameter data - the dictionary to look through
-@:parameter outfile - the file to write to 
+@:parameter outfile - the file to write to
 makeMain generates the vm_host/main.yml file
 it pulls two sets of information; dictionary data and proxy data
 """
@@ -122,7 +122,9 @@ def makeMain(data, outfile):
         "skip_image_downloading": veos.get("skip_image_downloading"),
         "vm_console_base": veos.get("vm_console_base"),
         "memory": veos.get("memory"),
-        "max_fp_num": veos.get("max_fp_num")
+        "max_fp_num": veos.get("max_fp_num"),
+        "ptf_bp_ip": veos.get("ptf_bp_ip"),
+        "ptf_bp_ipv6": veos.get("ptf_bp_ipv6")
     }
     proxy = {
         "proxy_env": {
@@ -141,21 +143,21 @@ def makeMain(data, outfile):
 @:parameter data - the dictionary to look for (in this case: veos)
 @:parameter outfile - the file to write to
 generates /group_vars/vm_host/creds.yml
-pulls ansible_user, ansible_password, ansible_sudo_pass from vm_host_ansible into a dictionary
+pulls ansible_user, ansible_password, ansible_become_pass from vm_host_ansible into a dictionary
 """
 def makeVMHostCreds(data, outfile):
     veos = data
     result = {
         "ansible_user": veos.get("vm_host_ansible").get("ansible_user"),
         "ansible_password": veos.get("vm_host_ansible").get("ansible_password"),
-        "ansible_sudo_password": veos.get("vm_host_ansible").get("ansible_sudo_pass")
+        "ansible_become_pass": veos.get("vm_host_ansible").get("ansible_become_pass")
     }
     with open(outfile, "w") as toWrite:
         toWrite.write("---\n")
         yaml.dump(result, stream=toWrite, default_flow_style=False)
 
 """
-makeSonicLabDevices(data, outfile) 
+makeSonicLabDevices(data, outfile)
 @:parameter data - the dictionary to look through (devices dictionary)
 @:parameter outfile - the file to write to
 generates files/sonic_lab_devices.csv by pulling hostname, managementIP, hwsku, and type
@@ -190,14 +192,14 @@ def makeSonicLabDevices(data, outfile):
 
 
 """
-makeTestbed(data, outfile) 
+makeTestbed(data, outfile)
 @:parameter data - the dictionary to look through (devices dictionary)
 @:parameter outfile - the file to write to
 generates /testbed.csv by pulling confName, groupName, topo, ptf_image_name, ptf_ip, server, vm_base, dut, and comment
 error handling: checks if attribute values are None type or string "None"
 """
 def makeTestbed(data, outfile):
-    csv_columns = "# conf-name,group-name,topo,ptf_image_name,ptf_ip,server,vm_base,dut,ptf,comment"
+    csv_columns = "# conf-name,group-name,topo,ptf_image_name,ptf,ptf_ip,server,vm_base,dut,comment"
     topology = data
     csv_file = outfile
 
@@ -236,7 +238,7 @@ def makeTestbed(data, outfile):
                 if not comment:
                     comment = ""
 
-                row = confName + "," + groupName + "," + topo + "," + ptf_image_name + "," + ptf_ip + "," + server + "," + vm_base + "," + dut + "," + ptf + "," + comment
+                row = confName + "," + groupName + "," + topo + "," + ptf_image_name + "," + ptf + "," + ptf_ip + "," + server + "," + vm_base + "," + dut + "," + comment
                 f.write(row + "\n")
     except IOError:
         print("I/O error: issue creating testbed.csv")
@@ -245,9 +247,9 @@ def makeTestbed(data, outfile):
 """
 makeSonicLabLinks(data, outfile)
 @:parameter data - the dictionary to look through (devices dictionary)
-@:parameter outfile - the file to write to  
+@:parameter outfile - the file to write to
 generates /files/sonic_lab_links.csv by pulling startPort, endPort, bandWidth, vlanID, vlanMode
-error handling: checks if attribute values are None type or string "None" 
+error handling: checks if attribute values are None type or string "None"
 """
 def makeSonicLabLinks(data, outfile):
     csv_columns = "StartDevice,StartPort,EndDevice,EndPort,BandWidth,VlanID,VlanMode"
@@ -308,7 +310,7 @@ def makeEOSCreds(data, outfile):
 """
 makeFanout_secrets(data, outfile)
 @:parameter data - reads from devices dictionary
-@:parameter outfile - the file to write to 
+@:parameter outfile - the file to write to
 Makes /group_vars/fanout/secrets.yml
 Finds the fanout secret credentials by using "fanout" as the value to search for under device_type
 Under github and personal topology configuration, there is only one designated fanout switch credential
@@ -428,7 +430,7 @@ def makeLab(data, devices, testbed, outfile):
 """
 makeVeos(data, veos, devices, outfile)
 @:parameter data - reads from either veos-groups, this helps separate the function into 3 components; children, host, vars
-@:parameter veos - reads from either veos 
+@:parameter veos - reads from either veos
 @:parameter devices - reads from devices
 @:parameter outfile - writes to veos
 """
@@ -487,8 +489,8 @@ def makeHostVar(data):
 
 """
 updateDockerRegistry
-@:parameter outfile - the file to write to 
-hard codes the docker registry to search locally rather than externally 
+@:parameter outfile - the file to write to
+hard codes the docker registry to search locally rather than externally
 """
 def updateDockerRegistry(docker_registry, outfile):
     if (not docker_registry.get("docker_registry_host")) or (not docker_registry.get("docker_registry_username")) or (not docker_registry.get("docker_registry_password")):

ansible/config_sonic_basedon_testbed.yml

@@ -41,12 +41,17 @@
         testbed_file: testbed.csv
       when: testbed_file is not defined
 
+    - name: Set default dut index
+      set_fact:
+        dut_index: 0
+      when: dut_index is not defined
+
     - name: Gathering testbed information
       test_facts: testbed_name="{{ testbed_name }}" testbed_file="{{ testbed_file }}"
       delegate_to: localhost
 
     - fail: msg="The DUT you are trying to run test does not belongs to this testbed"
-      when: testbed_facts['dut'] != inventory_hostname
+      when: testbed_facts['duts'][dut_index] != inventory_hostname
 
     - name: set testbed_type
       set_fact:
@@ -144,6 +149,49 @@
         become: true
         when: stat_result.stat.exists is defined and stat_result.stat.exists
 
+      - name: Init account key and proxy
+        set_fact:
+            core_key: ""
+            core_proxy: ""
+
+      - name: read account key
+        set_fact:
+            core_key: "{{ corefile_uploader['azure_sonic_core_storage']['account_key'] }}"
+        when: corefile_uploader['azure_sonic_core_storage']['account_key'] is defined
+
+      - name: read https proxy
+        set_fact:
+            core_proxy: "{{ corefile_uploader['env']['https_proxy'] }}"
+        when: corefile_uploader['env']['https_proxy'] is defined
+
+      - name: Put secret in core_analyzer.rc.json
+        lineinfile:
+            name: /etc/sonic/core_analyzer.rc.json
+            regexp: '(^.*)account_key'
+            line: '\1account_key": "{{ core_key }}",'
+            backrefs: yes
+        become: true
+        when: core_key != ""
+
+      - name: Put https-proxy in core_analyzer.rc.json
+        lineinfile:
+            name: /etc/sonic/core_analyzer.rc.json
+            regexp: '(^.*)https_proxy'
+            line: '\1https_proxy": "{{ core_proxy }}"'
+            backrefs: yes
+        become: true
+        when: core_proxy != ""
+
+      - name: enable core uploader service
+        become: true
+        command: systemctl enable core_uploader.service
+        when: core_key != ""
+
+      - name: start core uploader service
+        become: true
+        command: systemctl start core_uploader.service
+        when: core_key != ""
+
       - name: Replace snmp community string
         lineinfile:
             name: /etc/sonic/snmp.yml

ansible/doc/README.testbed.VsSetup.md

@@ -126,7 +126,7 @@ from the ```sonic-mgmt``` container. Then, test you can sudo without password pr
 ```
 $ ./testbed-cli.sh -m veos.vtb -n 4 start-vms server_1 password.txt
 ```
-  - please note: Here "password.txt" is the ansible vault password file name/path. Ansible allows user use ansible vault to encrypt password files. By default, this shell script require a password file. If you are not using ansible vault, just create an empty file and pass the filename to the command line. The file name and location is created and maintained by user.
+  - please note: Here "password.txt" is the ansible vault password file name/path. Ansible allows user use ansible vault to encrypt password files. By default, this shell script require a password file. If you are not using ansible vault, just create a file with a dummy pasword and pass the filename to the command line. The file name and location is created and maintained by user.
 
 Check that all VMs are up and running, and the passwd is ```123456```
 ```

ansible/doc/README.testbed.cEOS.md

@@ -0,0 +1,131 @@
+# cEOS
+
+This document discusses how to use cEOS as DUT neighbor device.
+
+cEOS is the container-based EOS. All the software running inside
+the container. Compared with vEOS, cEOS has much smaller memory
+footprint.
+
+Follow [instruction](README.testbed.VsSetup.md) to setup cEOS testbed.
+
+In below example, there are four cEOS containers.
+
+```
+lgh@jenkins-worker-15:~$ docker ps
+CONTAINER ID        IMAGE                                                        COMMAND                  CREATED             STATUS              PORTS               NAMES
+fe48c207a51c        ceosimage:4.23.2F-1                                          "/sbin/init systemd.…"   8 days ago          Up 8 days                               ceos_vms6-1_VM0103
+52297010e66a        ceosimage:4.23.2F-1                                          "/sbin/init systemd.…"   8 days ago          Up 8 days                               ceos_vms6-1_VM0102
+8dd95269b312        ceosimage:4.23.2F-1                                          "/sbin/init systemd.…"   8 days ago          Up 8 days                               ceos_vms6-1_VM0101
+3a50dd481bfb        ceosimage:4.23.2F-1                                          "/sbin/init systemd.…"   8 days ago          Up 8 days                               ceos_vms6-1_VM0100
+b91b48145def        debian:jessie                                                "bash"                   8 days ago          Up 8 days                               net_vms6-1_VM0103
+d1ff26d84249        debian:jessie                                                "bash"                   8 days ago          Up 8 days                               net_vms6-1_VM0102
+1489f52b9617        debian:jessie                                                "bash"                   8 days ago          Up 8 days                               net_vms6-1_VM0101
+ce1214a008ed        debian:jessie                                                "bash"                   8 days ago          Up 8 days                               net_vms6-1_VM0100
+```
+
+## Resource consumption
+
+A cEOS containers consumes around 1G memory.
+
+```
+lgh@jenkins-worker-15:~$ docker stats --no-stream
+CONTAINER ID        NAME                 CPU %               MEM USAGE / LIMIT     MEM %               NET I/O             BLOCK I/O           PIDS          6
+fe48c207a51c        ceos_vms6-1_VM0103   2.04%               970.9MiB / 125.9GiB   0.75%               0B / 0B             365MB / 55.8GB      138
+52297010e66a        ceos_vms6-1_VM0102   2.19%               965.4MiB / 125.9GiB   0.75%               0B / 0B             237MB / 55.6GB      139
+8dd95269b312        ceos_vms6-1_VM0101   1.93%               980.9MiB / 125.9GiB   0.76%               0B / 0B             300MB / 55.9GB      138
+3a50dd481bfb        ceos_vms6-1_VM0100   2.05%               970.2MiB / 125.9GiB   0.75%               0B / 0B             365MB / 56.1GB      138
+```
+
+## Network Setup
+
+We first create a base container `net_${testbed_name}_${vm_name}`, inject six ethernet ports into the base container, 
+and then start cEOS `ceos_${testbed_name}_${vm_name}` container on top of the base container. The six ethernet ports
+are used for
+- 1 management port
+- 4 front panel ports to DUT
+- 1 backplane port to PTF docker
+
+```
+         +------------+                      +----+
+         |  cEOS  Ma0 +--------- VM0100-m ---+ br |
+         |            |                      +----+
+         |            |
+         |            |                      +--------------+
+         |        Et1 +----------VM0100-t0---+  br-VM0100-0 |
+         |            |                      +--------------+
+         |            |
+         |            |                      +--------------+
+         |        Et2 +----------VM0100-t1---+  br-VM0100-1 |
+         |            |                      +--------------+
+         |            |
+         |            |                      +--------------+
+         |        Et3 +----------VM0100-t2---+  br-VM0100-2 |
+         |            |                      +--------------+
+         |            |
+         |            |                      +--------------+
+         |        Et4 +----------VM0100-t3---+  br-VM0100-3 |
+         |            |                      +--------------+
+         |            |
+         |            |                       +--------------+
+         |        Et5 +----------VM0100-back--+  br-b-vms6-1 |
+         |            |                       +--------------+
+         +------------+
+```
+
+## Configuration
+
+The `/mnt/flash` in cEOS container is mount to `/data/ceos/ceos_${testbed_name}_${vm_name}` on the host. The `/mnt/flash`
+contiains the configuration file and logs.
+
+```
+lgh@jenkins-worker-15:~$ ls -l /data/ceos/ceos_vms6-1_VM0100/
+total 40
+-rw-rw-r--+ 1 root root  924 Mar 31 07:35 AsuFastPktTransmit.log
+drwxrwxr-x+ 2 root root 4096 Mar 31 03:31 Fossil
+-rw-rw-r--+ 1 root root  568 Mar 31 07:35 SsuRestore.log
+-rw-rw-r--+ 1 root root  568 Mar 31 07:35 SsuRestoreLegacy.log
+drwxr-xr-x+ 4  897   88 4096 Mar 31 07:35 archive
+drwxrwx---+ 3 root root 4096 Mar 18 06:12 debug
+drwxrwxr-x+ 2 root root 4096 Mar 18 06:12 fastpkttx.backup
+-rw-rw-r--+ 1 root root  180 Mar 31 07:35 kickstart-config
+drwxrwxr-x+ 3 root root 4096 Apr  8 09:11 persist
+-rw-rwxr--+ 1 root root 1915 Mar 18 06:12 startup-config
+```
+
+## Login
+
+There are two ways to get into cEOS container
+
+1. docker exec
+```
+lgh@jenkins-worker-15:~$ docker exec -it ceos_vms6-1_VM0100 Cli
+ARISTA01T1>show int status
+Port       Name      Status       Vlan     Duplex Speed  Type            Flags Encapsulation
+Et1                  connected    in Po1   full   unconf EbraTestPhyPort                    
+Et2                  connected    1        full   unconf EbraTestPhyPort                    
+Et3                  connected    1        full   unconf EbraTestPhyPort                    
+Et4                  connected    1        full   unconf EbraTestPhyPort                    
+Et5        backplane connected    routed   full   unconf EbraTestPhyPort                    
+Ma0                  connected    routed   full   10G    10/100/1000                        
+Po1                  connected    routed   full   unconf N/A    
+
+ARISTA01T1>
+```
+
+2. ssh
+```
+lgh@jenkins-worker-15:~$ ssh [email protected]
+Password: 
+ARISTA01T1>show int status
+Port       Name      Status       Vlan     Duplex Speed  Type            Flags Encapsulation
+Et1                  connected    in Po1   full   unconf EbraTestPhyPort                    
+Et2                  connected    1        full   unconf EbraTestPhyPort                    
+Et3                  connected    1        full   unconf EbraTestPhyPort                    
+Et4                  connected    1        full   unconf EbraTestPhyPort                    
+Et5        backplane connected    routed   full   unconf EbraTestPhyPort                    
+Ma0                  connected    routed   full   10G    10/100/1000                        
+Po1                  connected    routed   full   unconf N/A                                
+
+ARISTA01T1>
+```
+