[sonic-config-engine] Clean up dependencies, pin versions; install Python 3 package in Buster container

dockers/docker-config-engine-buster/Dockerfile.j2

@@ -6,22 +6,25 @@ ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update         && \
     apt-get install -y        \
-        # Dependencies for sonic-cfggen
-        python-lxml           \
-        python-yaml           \
-        python-bitarray       \
+        build-essential       \
         python-pip            \
+        python3-pip           \
         python-dev            \
+        python3-dev           \
         apt-utils             \
-        python-setuptools
+        python-setuptools     \
+        python3-setuptools
 
-RUN pip install --upgrade pip
+RUN pip2 install --upgrade pip
+RUN pip3 install --upgrade pip
+RUN apt-get purge -y python-pip python3-pip
 
-RUN pip install             \
-        netaddr             \
-        ipaddr              \
-        jinja2              \
-        pyangbind==0.6.0
+# For sonic-config-engine Python 3 package
+# Explicitly install pyangbind here, as pyangbind causes enum34 to be installed.
+# enum34 causes Python 're' package to not work properly as it redefines an incompatible enum.py module
+# https://github.com/robshakir/pyangbind/issues/232
+RUN pip3 install pyangbind==0.8.1
+RUN pip3 uninstall -y enum34
 
 {% if docker_config_engine_buster_debs.strip() %}
 # Copy locally-built Debian package dependencies
@@ -41,8 +44,9 @@ RUN pip install             \
 
 ## Clean up
 RUN apt-get purge -y               \
-        python-pip                 \
-        python-dev              && \
+        python-dev                 \
+        python3-dev                \
+        build-essential         && \
     apt-get clean -y            && \
     apt-get autoclean -y        && \
     apt-get autoremove -y       && \

dockers/docker-config-engine-stretch/Dockerfile.j2

@@ -7,21 +7,13 @@ ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update         && \
     apt-get install -y        \
         # Dependencies for sonic-cfggen
-        python-lxml           \
-        python-yaml           \
-        python-bitarray       \
+        build-essential       \
         python-pip            \
         python-dev            \
         python-setuptools
 
 RUN pip install --upgrade pip
 
-RUN pip install             \
-        netaddr             \
-        ipaddr              \
-        jinja2              \
-        pyangbind==0.5.10
-
 {% if docker_config_engine_stretch_debs.strip() %}
 # Copy locally-built Debian package dependencies
 {{ copy_files("debs/", docker_config_engine_stretch_debs.split(' '), "/debs/") }}
@@ -40,6 +32,7 @@ RUN pip install             \
 
 ## Clean up
 RUN apt-get purge -y               \
+        build-essential            \
         python-pip                 \
         python-dev              && \
     apt-get clean -y            && \

dockers/docker-config-engine/Dockerfile.j2

@@ -6,11 +6,11 @@ ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update
 
 # Dependencies for sonic-cfggen
-RUN apt-get install -y python-lxml python-yaml python-bitarray python-pip python-dev
+RUN apt-get install -y build-essential python-pip python-dev
 
 RUN pip install --upgrade pip
 
-RUN pip install netaddr ipaddr jinja2 pyangbind==0.5.10
+RUN apt-get purge -y python-pip
 
 {% if docker_config_engine_debs.strip() %}
 COPY \
@@ -43,5 +43,5 @@ python-wheels/{{ whl }}{{' '}}
 {%- endif -%}
 
 ## Clean up
-RUN apt-get purge -y python-pip python-dev; apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y
+RUN apt-get purge -y build-essential python-dev; apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y
 RUN rm -rf /debs /python-wheels

dockers/docker-orchagent/Dockerfile.j2

@@ -4,7 +4,7 @@ FROM docker-config-engine-buster
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf
 
-## Make apt-get non-interactive
+# Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update &&       \
@@ -18,10 +18,14 @@ RUN apt-get update &&       \
         libmnl0             \
         bridge-utils        \
         conntrack           \
-        ndppd
+        ndppd               \
+        # Needed for installing netifaces Python package
+        build-essential     \
+        python-dev          \
+        python3-dev
 
 {% if ( CONFIGURED_ARCH == "armhf" or CONFIGURED_ARCH == "arm64" ) %}
-## Fix for gcc/python not found in arm docker
+# Fix for gcc/python not found in arm docker
 RUN apt-get install -f -y python2.7 python2.7-dev
 RUN apt-get install -y gcc-6
 {% endif %}
@@ -53,10 +57,14 @@ RUN apt-get remove -y gcc-6
 {{ install_debian_packages(docker_orchagent_debs.split(' ')) }}
 {%- endif %}
 
-## Clean up
-RUN apt-get clean -y        && \
-    apt-get autoclean -y    && \
-    apt-get autoremove -y   && \
+# Clean up
+RUN apt-get purge -y          \
+        build-essential       \
+        python-dev            \
+        python3-dev        && \
+    apt-get clean -y       && \
+    apt-get autoclean -y   && \
+    apt-get autoremove -y  && \
     rm -rf /debs
 
 COPY ["files/arp_update", "/usr/bin"]
@@ -68,7 +76,7 @@ COPY ["supervisord.conf", "/etc/supervisor/conf.d/"]
 COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
 COPY ["critical_processes", "/etc/supervisor/"]
 
-## Copy all Jinja2 template files into the templates folder
+# Copy all Jinja2 template files into the templates folder
 COPY ["*.j2", "/usr/share/sonic/templates/"]
 
 ENTRYPOINT ["/usr/bin/docker-init.sh"]

dockers/docker-sonic-mgmt-framework/Dockerfile.j2

@@ -7,7 +7,7 @@ RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%s
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update && \
-    apt-get install -y g++ python-dev
+    apt-get install -y g++ python-dev libxml2
 
 RUN pip install connexion==1.1.15 \
 		setuptools==21.0.0 \

dockers/dockerfile-macros.j2

@@ -6,7 +6,7 @@ RUN dpkg_apt() { [ -f $1 ] && { dpkg -i $1 || apt-get -y install -f; } || return
 {%- endmacro %}
 
 {% macro install_python2_wheels(packages) -%}
-RUN cd /python-wheels/ && pip install {{ packages | join(' ') }}
+RUN cd /python-wheels/ && pip2 install {{ packages | join(' ') }}
 {%- endmacro %}
 
 {% macro install_python3_wheels(packages) -%}

files/build_templates/sonic_debian_extension.j2

@@ -93,10 +93,7 @@ sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/iptables_*.deb || \
 # Install dependencies for SONiC config engine
 sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install \
     python-dev     \
-    python3-dev    \
-    python-lxml    \
-    python-yaml    \
-    python-bitarray
+    python3-dev
 
 # Install Python client for Redis
 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install "redis==3.5.3"

platform/vs/docker-sonic-vs/Dockerfile.j2

@@ -61,20 +61,20 @@ RUN curl -o redis-server_6.0.6-1~bpo10+1_amd64.deb "https://sonicstorage.blob.co
 RUN dpkg -i redis-tools_6.0.6-1~bpo10+1_amd64.deb redis-server_6.0.6-1~bpo10+1_amd64.deb || apt-get install -f
 RUN rm redis-tools_6.0.6-1~bpo10+1_amd64.deb redis-server_6.0.6-1~bpo10+1_amd64.deb
 
-RUN pip install setuptools
+RUN pip2 install setuptools
 RUN pip3 install setuptools
-RUN pip install py2_ipaddress
-RUN pip install six
-RUN pip install pyroute2==0.5.3 netifaces==0.10.7
-RUN pip install monotonic==1.5
-RUN pip install urllib3
-RUN pip install requests
-RUN pip install crontab
+RUN pip2 install py2_ipaddress
+RUN pip2 install six
+RUN pip2 install pyroute2==0.5.3 netifaces==0.10.7
+RUN pip2 install monotonic==1.5
+RUN pip2 install urllib3
+RUN pip2 install requests
+RUN pip2 install crontab
 
 # Install dependencies for Dynamic Port Breakout
-RUN pip install xmltodict==0.12.0
-RUN pip install jsondiff==1.2.0
-RUN pip install ijson==2.6.1
+RUN pip2 install xmltodict==0.12.0
+RUN pip2 install jsondiff==1.2.0
+RUN pip2 install ijson==2.6.1
 
 {% if docker_sonic_vs_debs.strip() -%}
 # Copy locally-built Debian package dependencies
@@ -109,7 +109,7 @@ COPY python-wheels/{{ whl }}  python-wheels/
 # install PKGs after copying all PKGs to avoid dependency failure
 # use py3 to find python3 package, which is forced by wheel as of now
 {%- for whl in docker_sonic_vs_whls.split(' ') %}
-RUN pip{% if 'py3' in whl %}3{% endif %} install python-wheels/{{ whl }}
+RUN pip{% if 'py3' in whl %}3{% else %}2{% endif %} install python-wheels/{{ whl }}
 {%- endfor %}
 {% endif %}
 

rules/docker-config-engine-buster.mk

@@ -4,8 +4,11 @@ DOCKER_CONFIG_ENGINE_BUSTER = docker-config-engine-buster.gz
 $(DOCKER_CONFIG_ENGINE_BUSTER)_PATH = $(DOCKERS_PATH)/docker-config-engine-buster
 
 $(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SWSSSDK_PY2)
+$(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SWSSSDK_PY3)
 $(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY2)
+$(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY3)
 $(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE_PY2)
+$(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE_PY3)
 $(DOCKER_CONFIG_ENGINE_BUSTER)_LOAD_DOCKERS += $(DOCKER_BASE_BUSTER)
 
 $(DOCKER_CONFIG_ENGINE_BUSTER)_DBG_DEPENDS = $($(DOCKER_BASE_BUSTER)_DBG_DEPENDS)

rules/sonic-config.mk

@@ -6,11 +6,13 @@ $(SONIC_CONFIG_ENGINE_PY2)_DEPENDS += $(SWSSSDK_PY2) $(SONIC_PY_COMMON_PY2)
 $(SONIC_CONFIG_ENGINE_PY2)_PYTHON_VERSION = 2
 SONIC_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE_PY2)
 
-
 # SONIC_CONFIG_ENGINE_PY3 package
 
 SONIC_CONFIG_ENGINE_PY3 = sonic_config_engine-1.0-py3-none-any.whl
 $(SONIC_CONFIG_ENGINE_PY3)_SRC_PATH = $(SRC_PATH)/sonic-config-engine
 $(SONIC_CONFIG_ENGINE_PY3)_DEPENDS += $(SWSSSDK_PY3) $(SONIC_PY_COMMON_PY3)
+# Synthetic dependency to avoid building the Python 2 and 3 packages
+# simultaneously and any potential conflicts which may arise
+$(SONIC_CONFIG_ENGINE_PY3)_DEPENDS += $(SONIC_CONFIG_ENGINE_PY2)
 $(SONIC_CONFIG_ENGINE_PY3)_PYTHON_VERSION = 3
 SONIC_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE_PY3)

sonic-slave-buster/Dockerfile.j2

@@ -57,10 +57,10 @@ RUN apt-get update && apt-get install -y \
         libtinyxml2-dev \
         python \
         python-pip \
+        python3-pip \
         libncurses5-dev \
         texinfo \
         dh-autoreconf \
-        python3-pip \
         doxygen \
         devscripts \
         git-buildpackage \
@@ -245,13 +245,6 @@ RUN apt-get update && apt-get install -y \
         python3-sphinx \
 # For sonic config engine testing
         python-dev \
-        python-lxml \
-        python3-lxml \
-        python-jinja2 \
-        python-netaddr \
-        python-ipaddr \
-        python-yaml \
-        python3-yaml \
 # For lockfile
         procmail \
 # For gtest
@@ -305,7 +298,14 @@ RUN apt-get update && apt-get install -y \
 # For SWI Tools
         python-m2crypto \
 # For build dtb
-	device-tree-compiler
+        device-tree-compiler \
+# For sonic-mgmt-framework
+        autoconf \
+        m4 \
+        libxml2-utils \
+        xsltproc \
+        python-lxml \
+        libexpat1-dev
 
 ## Config dpkg
 ## install the configuration file if it’s currently missing
@@ -334,29 +334,33 @@ RUN export VERSION=1.14.2 \
  && echo 'export PATH=$PATH:$GOROOT/bin' >> /etc/bash.bashrc \
  && rm go$VERSION.linux-*.tar.gz
 
+RUN pip2 install --upgrade pip
+RUN pip3 install --upgrade pip
+RUN apt-get purge -y python-pip python3-pip
+
 # For building Python packages
-RUN pip install setuptools==40.8.0
+RUN pip2 install setuptools==40.8.0
 RUN pip3 install setuptools==49.6.00
 
 # For building sonic-utilities
-RUN pip install fastentrypoints
+RUN pip2 install fastentrypoints
 RUN pip3 install fastentrypoints
 
 # For running Python unit tests
-RUN pip install pytest-runner==4.4
+RUN pip2 install pytest-runner==4.4
 RUN pip3 install pytest-runner==5.2
-RUN pip install mockredispy==2.9.3
+RUN pip2 install mockredispy==2.9.3
 RUN pip3 install mockredispy==2.9.3
 
 # For Python 2 unit tests, we need 'mock'. The last version of 'mock'
 # which supports Python 2 is 3.0.5. In Python 3, 'mock' is part of 'unittest'
 # in the standard library
-RUN pip install mock==3.0.5
+RUN pip2 install mock==3.0.5
 
 # For p4 build
-RUN pip install \
-        ctypesgen==1.0.2 \
-        crc16
+RUN pip2 install \
+         ctypesgen==1.0.2 \
+         crc16
 
 # For sonic config engine testing
 # Install pyangbind here, outside sonic-config-engine dependencies, as pyangbind causes enum34 to be installed.
@@ -366,31 +370,35 @@ RUN pip3 install pyangbind==0.8.1
 RUN pip3 uninstall -y enum34
 
 # For templating
-RUN pip install j2cli==0.3.10
+RUN pip2 install j2cli==0.3.10
 
-RUN pip3 install "PyYAML>=5.1"
+# For sonic-mgmt-framework
+RUN pip2 install "PyYAML==5.3.1"
+RUN pip3 install "PyYAML==5.3.1"
+RUN pip2 install "lxml==4.6.1"
+RUN pip3 install "lxml==4.6.1"
 
 # For sonic-platform-common testing
 RUN pip3 install redis
 
 # For supervisor build
-RUN pip install meld3
+RUN apt-get install python-meld3
 
 # For vs image build
-RUN pip install pexpect==4.6.0
+RUN pip2 install pexpect==4.6.0
 
 # For sonic-swss-common testing
-RUN pip install Pympler==0.8
+RUN pip2 install Pympler==0.8
 
 # For sonic_yang_mgmt build
-RUN pip install ijson==2.6.1
+RUN pip2 install ijson==2.6.1
 RUN pip3 install ijson==2.6.1
-RUN pip install jsondiff==1.2.0
-RUN pip install xmltodict==0.12.0
-RUN pip install pyang==2.1.1
+RUN pip2 install jsondiff==1.2.0
+RUN pip2 install xmltodict==0.12.0
+RUN pip2 install pyang==2.1.1
 
 # For mgmt-framework build
-RUN pip install mmh3
+RUN pip2 install mmh3
 RUN apt-get install -y xsltproc
 
 # Install dependencies for isc-dhcp-relay build
@@ -434,4 +442,4 @@ RUN echo "DOCKER_OPTS=\"--experimental --storage-driver=vfs\"" >> /etc/default/d
 RUN update-alternatives --set iptables /usr/sbin/iptables-legacy
 
 # Install swi tools
-RUN python -m pip install git+https://github.com/aristanetworks/swi-tools.git@d51761ec0bb93c73039233f3c01ed48235ffad00
+RUN pip2 install git+https://github.com/aristanetworks/swi-tools.git@d51761ec0bb93c73039233f3c01ed48235ffad00