[python-click] No longer build our own package, let pip/setuptools in…

…stall vanilla (sonic-net#5549) We were building our own python-click package because we needed features/bug fixes available as of version 7.0.0, but the most recent version available from Debian was in the 6.x range. "Click" is needed for building/testing and installing sonic-utilities. Now that we are building sonic-utilities as a wheel, with Click specified as a dependency in the setup.py file, setuptools will install a more recent version of Click in the sonic-slave-buster container when building the package, and pip will install a more recent version of Click in the host OS of SONiC when installing the sonic-utilities package. Also, we don't need to worry about installing the Python 2 or 3 version of the package, as the proper one will be installed as necessary.
santhosh-kt · Feb 25, 2021 · 9b04069 · 9b04069
1 parent ba83c3d
commit 9b04069
Show file tree

Hide file tree

Showing 14 changed files with 40 additions and 121 deletions.
diff --git a/dockers/docker-config-engine-buster/Dockerfile.j2 b/dockers/docker-config-engine-buster/Dockerfile.j2
@@ -12,7 +12,6 @@ RUN apt-get update         && \
         python-bitarray       \
         python-pip            \
         python-dev            \
-        python-natsort        \
         apt-utils             \
         python-setuptools
 

diff --git a/dockers/docker-config-engine-stretch/Dockerfile.j2 b/dockers/docker-config-engine-stretch/Dockerfile.j2
@@ -12,7 +12,6 @@ RUN apt-get update         && \
         python-bitarray       \
         python-pip            \
         python-dev            \
-        python-natsort        \
         python-setuptools
 
 RUN pip install --upgrade pip

diff --git a/dockers/docker-config-engine/Dockerfile.j2 b/dockers/docker-config-engine/Dockerfile.j2
@@ -6,7 +6,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update
 
 # Dependencies for sonic-cfggen
-RUN apt-get install -y python-lxml python-yaml python-bitarray python-pip python-dev python-natsort
+RUN apt-get install -y python-lxml python-yaml python-bitarray python-pip python-dev
 
 RUN pip install --upgrade pip
 

diff --git a/files/build_templates/sonic_debian_extension.j2 b/files/build_templates/sonic_debian_extension.j2
@@ -93,6 +93,7 @@ sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/iptables_*.deb || \
 # Install dependencies for SONiC config engine
 sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install \
     python-dev     \
+    python3-dev    \
     python-lxml    \
     python-yaml    \
     python-bitarray
@@ -184,14 +185,12 @@ sudo cp {{platform_common_py3_wheel_path}} $FILESYSTEM_ROOT/$PLATFORM_COMMON_PY3
 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install $PLATFORM_COMMON_PY3_WHEEL_NAME
 sudo rm -rf $FILESYSTEM_ROOT/$PLATFORM_COMMON_PY3_WHEEL_NAME
 
-# Install Debian packages and their dependencies which are needed by sonic-utilities
-sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f python-m2crypto
+# Install prerequisites needed for installing the Python m2crypto package, used by sonic-utilities
+# These packages can be uninstalled after intallation
+sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install build-essential libssl-dev swig
 
-# Install built Python Click package (and its dependencies via 'apt-get -y install -f')
-# Do this before installing sonic-utilities so that it doesn't attempt to install
-# an older version as part of its dependencies
-sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/python-click*_all.deb || \
-    sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
+# Install prerequisites needed for using the Python m2crypto package, used by sonic-utilities
+sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install openssl
 
 # Install SONiC Utilities Python 2 package
 SONIC_UTILITIES_PY2_WHEEL_NAME=$(basename {{sonic_utilities_py2_wheel_path}})
@@ -574,7 +573,8 @@ sudo cp $BUILD_TEMPLATES/mgmt-framework.timer $FILESYSTEM_ROOT_USR_LIB_SYSTEMD_S
 echo "mgmt-framework.timer" | sudo tee -a $GENERATED_SERVICE_FILE
 {% endif %}
 
-sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get purge -y python-dev
+sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get purge -y python-dev python3-dev
+sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get purge -y build-essential libssl-dev swig
 sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get clean -y
 sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get autoremove -y
 

diff --git a/platform/p4/docker-sonic-p4/Dockerfile.j2 b/platform/p4/docker-sonic-p4/Dockerfile.j2
@@ -40,12 +40,18 @@ RUN apt-get install -y net-tools \
                        iproute \
                        libpython2.7 \
                        grub2-common \
-                       python-click \
-                       python-natsort \
-                       python-tabulate \
                        bash-completion \
                        libelf1 \
-                       libmnl0
+                       libmnl0 \
+                       # For installing Python m2crypto package
+                       # (these can be uninstalled after installation)
+                       build-essential \
+                       python-dev \
+                       python3-dev \
+                       libssl-dev \
+                       swig \
+                       # For using Python m2crypto package
+                       openssl
 
 RUN pip install setuptools
 RUN pip install py2_ipaddress
@@ -62,6 +68,8 @@ debs/{{ deb }}{{' '}}
 {%- endfor %}
 
 ## Clean up
+RUN apt-get purge -y build-essential libssl-dev swig
+RUN apt-get purge -y python-dev python3-dev
 RUN apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y
 RUN rm -rf /debs
 

diff --git a/platform/vs/docker-sonic-vs/Dockerfile.j2 b/platform/vs/docker-sonic-vs/Dockerfile.j2
@@ -32,9 +32,6 @@ RUN apt-get install -y net-tools \
                        libc-ares2 \
                        iproute2 \
                        grub2-common \
-                       python-click \
-                       python-natsort \
-                       python-tabulate \
                        bash-completion \
                        libelf1 \
                        libmnl0 \
@@ -47,8 +44,16 @@ RUN apt-get install -y net-tools \
                        iptables \
                        python3-pip \
                        jq \
-                       python-m2crypto \
-                       libzmq5
+                       libzmq5 \
+                       # For installing Python m2crypto package
+                       # (these can be uninstalled after installation)
+                       build-essential \
+                       python-dev \
+                       python3-dev \
+                       libssl-dev \
+                       swig \
+                       # For using Python m2crypto package
+                       openssl
 
 # install redis-server
 RUN curl -o redis-tools_6.0.6-1~bpo10+1_amd64.deb "https://sonicstorage.blob.core.windows.net/packages/redis/redis-tools_6.0.6-1~bpo10+1_amd64.deb?sv=2015-04-05&sr=b&sig=73zbmjkf3pi%2Bn0R8Hy7CWT2EUvOAyzM5aLYJWCLySGM%3D&se=2030-09-06T19%3A44%3A59Z&sp=r"
@@ -111,6 +116,8 @@ RUN pip{% if 'py3' in whl %}3{% endif %} install python-wheels/{{ whl }}
 {% endif %}
 
 # Clean up
+RUN apt-get purge -y build-essential libssl-dev swig
+RUN apt-get purge -y python-dev python3-dev
 RUN apt-get clean -y
 RUN apt-get autoclean -y
 RUN apt-get autoremove -y

diff --git a/rules/python-click.dep b/rules/python-click.dep
diff --git a/rules/python-click.mk b/rules/python-click.mk
diff --git a/sonic-slave-buster/Dockerfile.j2 b/sonic-slave-buster/Dockerfile.j2
@@ -270,17 +270,6 @@ RUN apt-get update && apt-get install -y \
         texlive-latex-extra \
         texlive-latex-recommended \
         iproute2 \
-# For python-click build
-        python-sphinx \
-        python3-pallets-sphinx-themes \
-        python-docutils \
-        python3-all \
-        python3-setuptools \
-        python3-sphinx \
-        python3-docutils \
-        python3-requests \
-        python3-pytest \
-        python3-colorama \
 # For bash
         texi2html \
 # For initramfs
@@ -349,6 +338,10 @@ RUN export VERSION=1.14.2 \
 RUN pip install setuptools==40.8.0
 RUN pip3 install setuptools==49.6.00
 
+# For building sonic-utilities
+RUN pip install fastentrypoints
+RUN pip3 install fastentrypoints
+
 # For running Python unit tests
 RUN pip install pytest-runner==4.4
 RUN pip3 install pytest-runner==5.2
@@ -375,11 +368,6 @@ RUN pip3 uninstall -y enum34
 # For templating
 RUN pip install j2cli==0.3.10
 
-# Remove python-click 6.6
-RUN apt-get purge -y python-click
-# For sonic utilities testing
-RUN pip install click natsort tabulate netifaces==0.10.7 fastentrypoints
-
 RUN pip3 install "PyYAML>=5.1"
 
 # For sonic-platform-common testing

diff --git a/sonic-slave-jessie/Dockerfile.j2 b/sonic-slave-jessie/Dockerfile.j2
@@ -246,16 +246,6 @@ RUN apt-get update && apt-get install -y \
         lynx \
         texlive-latex-extra \
         texlive-latex-recommended \
-# For python-click build
-        python-sphinx \
-        python-docutils \
-        python3-all \
-        python3-setuptools \
-        python3-sphinx \
-        python3-docutils \
-        python3-requests \
-        python3-pytest \
-        python3-colorama \
 # For bash
         texi2html \
 # For initramfs
@@ -310,9 +300,6 @@ RUN pip install --force-reinstall --upgrade "jinja2>=2.10"
 # For templating (requiring jinja2)
 RUN pip install j2cli==0.3.10
 
-# For sonic utilities testing
-RUN pip install click natsort tabulate netifaces==0.10.7 fastentrypoints
-
 # For supervisor build
 RUN pip install meld3 mock
 

diff --git a/sonic-slave-stretch/Dockerfile.j2 b/sonic-slave-stretch/Dockerfile.j2
@@ -266,16 +266,6 @@ RUN apt-get update && apt-get install -y \
         texlive-latex-extra \
         texlive-latex-recommended \
         iproute2 \
-# For python-click build
-        python-sphinx \
-        python-docutils \
-        python3-all \
-        python3-setuptools \
-        python3-sphinx \
-        python3-docutils \
-        python3-requests \
-        python3-pytest \
-        python3-colorama \
 # For bash
         texi2html \
 # For initramfs
@@ -358,11 +348,6 @@ RUN pip3 uninstall -y enum34
 # For templating
 RUN pip install j2cli==0.3.10
 
-# Remove python-click 6.6
-RUN apt-get purge -y python-click
-# For sonic utilities testing
-RUN pip install click natsort tabulate netifaces==0.10.7 fastentrypoints
-
 # For sonic snmpagent mock testing
 RUN pip3 install mockredispy==2.9.3
 

diff --git a/src/python-click/.gitignore b/src/python-click/.gitignore
diff --git a/src/python-click/Makefile b/src/python-click/Makefile
diff --git a/src/sonic-py-common/setup.py b/src/sonic-py-common/setup.py
@@ -1,7 +1,7 @@
 from setuptools import setup
 
 dependencies = [
-    'natsort',
+    'natsort==6.2.1', # 6.2.1 is the last version which supports Python 2
     'pyyaml',
     'swsssdk>=2.0.1',
 ]
@@ -28,7 +28,8 @@
         'sonic_py_common',
     ],
     setup_requires= [
-        'pytest-runner'
+        'pytest-runner',
+        'wheel'
     ],
     tests_require=[
         'pytest',