{Packaging} Optimize Linux package and docker image by removing py file

Dockerfile

@@ -52,7 +52,11 @@ COPY . /azure-cli
 
 # 1. Build packages and store in tmp dir
 # 2. Install the cli and the other command modules that weren't included
-RUN ./scripts/install_full.sh \
+
+# Python image has build-in env $PYTHON_VERSION=3.10.10.
+# `ARG PYTHON_VERSION="3.10"` works on ARM64, but it can't override the default value on AMD64.
+RUN ./scripts/install_full.sh && python ./scripts/trim_sdk.py \
+ && python ./scripts/use_pyc.py /usr/local/lib/python${PYTHON_VERSION:0:4}/site-packages/ \
  && cat /azure-cli/az.completion > ~/.bashrc \
  && runDeps="$( \
     scanelf --needed --nobanner --recursive /usr/local \

build_scripts/windows/scripts/build.cmd

@@ -139,36 +139,8 @@ copy %REPO_ROOT%\build_scripts\windows\resources\CLI_LICENSE.rtf %BUILDING_DIR%
 copy %REPO_ROOT%\build_scripts\windows\resources\ThirdPartyNotices.txt %BUILDING_DIR%
 copy %REPO_ROOT%\NOTICE.txt %BUILDING_DIR%
 
-REM Remove .py and only deploy .pyc files
-pushd %BUILDING_DIR%\Lib\site-packages
-for /f %%f in ('dir /b /s *.pyc') do (
-    set PARENT_DIR=%%~df%%~pf..
-    echo !PARENT_DIR! | findstr /C:\Lib\site-packages\pip\ 1>nul
-    if !errorlevel! neq  0 (
-        REM Only take the file name without 'pyc' extension: e.g., (same below) __init__.cpython-310
-        set FILENAME=%%~nf
-        REM Truncate the '.cpython-310' postfix which is 12 chars long: __init__
-        REM https://stackoverflow.com/a/636391/2199657
-        set BASE_FILENAME=!FILENAME:~0,-12!
-        REM __init__.pyc
-        set pyc=!BASE_FILENAME!.pyc
-        REM Delete ..\__init__.py
-        del !PARENT_DIR!\!BASE_FILENAME!.py
-        REM Copy to ..\__init__.pyc
-        copy %%~f !PARENT_DIR!\!pyc! >nul
-        REM Delete __init__.pyc
-        del %%~f
-    ) ELSE (
-        echo --SKIP !PARENT_DIR! under pip
-    )
-)
-popd
-
-REM Remove __pycache__
-echo remove pycache
-for /d /r %BUILDING_DIR%\Lib\site-packages\pip %%d in (__pycache__) do (
-    if exist %%d rmdir /s /q "%%d"
-)
+REM replace .py with .pyc and remove __pycache__ dir to save space
+%BUILDING_DIR%\python.exe %REPO_ROOT%\scripts\use_pyc.py %BUILDING_DIR%\Lib\site-packages\ %PYTHON_VERSION%
 
 REM Remove dist-info
 echo remove dist-info

scripts/release/debian/build.sh

@@ -52,6 +52,11 @@ export PATH=$PATH:$WORKDIR/python_env/bin
 
 find ${WORKDIR}/src/ -name setup.py -type f | xargs -I {} dirname {} | grep -v azure-cli-testsdk | xargs pip3 install --no-deps
 pip3 install -r ${WORKDIR}/src/azure-cli/requirements.py3.$(uname).txt
+$WORKDIR/python_env/bin/python3 ${WORKDIR}/scripts/trim_sdk.py
+
+# replace .py with .pyc and remove __pycache__ dir to save space
+site_packages_dir=$(cd $WORKDIR/python_env/lib/python3.*/site-packages/ ; pwd)
+$WORKDIR/python_env/bin/python3 ${WORKDIR}/scripts/use_pyc.py $site_packages_dir
 
 # Create create directory for debian build
 mkdir -p $WORKDIR/debian

scripts/release/rpm/azure-cli.spec

@@ -52,6 +52,13 @@ source %{repo_path}/scripts/install_full.sh
 # cffi 1.15.0 doesn't work with RPM: https://foss.heptapod.net/pypy/cffi/-/issues/513
 %{python_cmd} -m pip install cffi==1.14.6
 
+# Remove unused SDK version
+%{python_cmd} %{repo_path}/scripts/trim_sdk.py
+
+# replace .py with .pyc and remove __pycache__ dir to save space
+site_packages_dir=$(cd %{buildroot}%{cli_lib_dir}/lib/python3.*/site-packages/ ; pwd)
+%{python_cmd} %{repo_path}/scripts/use_pyc.py $site_packages_dir
+
 deactivate
 
 # Fix up %{buildroot} appearing in some files...
@@ -78,11 +85,6 @@ AZ_INSTALLER=RPM PYTHONPATH=\"\$bin_dir/../lib64/az/lib/${python_version}/site-p
 " > %{buildroot}%{_bindir}/az
 rm %{buildroot}%{cli_lib_dir}/bin/python* %{buildroot}%{cli_lib_dir}/bin/pip*
 
-# Remove unused Network SDK API versions
-pushd %{buildroot}%{cli_lib_dir}/lib/${python_version}/site-packages/azure/mgmt/network/ > /dev/null
-rm -rf v2016_09_01 v2016_12_01 v2017_03_01 v2017_06_01 v2017_08_01 v2017_09_01 v2017_11_01 v2018_02_01 v2018_04_01 v2018_06_01 v2018_10_01 v2018_12_01 v2019_04_01 v2019_08_01 v2019_09_01 v2019_11_01 v2019_12_01 v2020_03_01
-popd > /dev/null
-
 # Set up tab completion
 mkdir -p %{buildroot}%{_sysconfdir}/bash_completion.d/
 cat %{repo_path}/az.completion > %{buildroot}%{_sysconfdir}/bash_completion.d/azure-cli

scripts/trim_sdk.py

@@ -70,6 +70,9 @@ def remove_unused_api_versions(resource_type):
         if resource_type in profile:
             # value is str like '2022-01-01' or SDKProfile
             value = profile[resource_type]
+            if value is None:
+                _LOGGER.info(f'{resource_type}\'s API version is None, skip')
+                return
             if isinstance(value, str):
                 used_api_versions.add(value)
             else:
@@ -108,7 +111,9 @@ def _print_folder_size(folder):
 
 
 def _get_all_sdks_to_trim():
-    resource_types = [k for k, v in AZURE_API_PROFILES['latest'].items() if k.import_prefix.startswith('azure.mgmt')]
+    # azure.mgmt.network has been removed in https://github.com/Azure/azure-cli/pull/25451
+    resource_types = [k for k, v in AZURE_API_PROFILES['latest'].items() if k.import_prefix.startswith('azure.mgmt')
+                      and k.import_prefix != 'azure.mgmt.network']
     return resource_types
 
 
@@ -155,7 +160,7 @@ def main():
     _print_folder_size(mgmt_sdk_dir)
 
     # Removed unused API versions
-    resource_types = _get_biggest_sdks_to_trim()
+    resource_types = _get_all_sdks_to_trim()
 
     for r in resource_types:
         remove_unused_api_versions(r)

scripts/use_pyc.py

@@ -0,0 +1,81 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+import logging
+import glob
+import os
+import platform
+import re
+import sys
+from pathlib import Path
+import shutil
+
+_LOGGER = logging.getLogger(__name__)
+
+
+def calculate_folder_size(start_path):
+    """Calculate total size of a folder and file count."""
+    # https://stackoverflow.com/questions/1392413/calculating-a-directorys-size-using-python
+    total_size = 0
+    total_count = 0
+    for dirpath, dirnames, filenames in os.walk(start_path):
+        for f in filenames:
+            fp = os.path.join(dirpath, f)
+            # skip if it is symbolic link
+            if not os.path.islink(fp):
+                total_count += 1
+                total_size += os.path.getsize(fp)
+
+    return total_size, total_count
+
+
+def _print_folder_size(folder):
+    size, count = calculate_folder_size(folder)
+    size_in_mb = size / 1048576  # 1 MB = 1024 * 1024 B = 1048576 B
+    _LOGGER.info(f"{size_in_mb:.2f} MB, {count} files")
+
+
+def main(folder, version=None):
+    _LOGGER.info(f'Replace .py with .pyc, base folder: {folder}')
+    _print_folder_size(folder)
+    if version is None:
+        version = re.search(r'python(\d\.\d+)', folder).group(1)
+    else:
+        # 3.10.10
+        version = '.'.join(version.split('.')[:2])
+    # invoke==1.2.0 has a weird file: invoke/completion/__pycache__/__init__.cpython-36.pyc
+    # define pyc suffix to skip it
+    pyc_suffix = f'cpython-{version.replace(".", "")}.pyc'
+    _LOGGER.info(f'pyc suffix: {pyc_suffix}')
+    for file in glob.glob(f'{folder}/**/__pycache__/*{pyc_suffix}', recursive=True):
+        # If pip's py files are also removed, the error is raised when installing some packages.
+        # See https://github.com/Azure/azure-cli/pull/25801 for details.
+        if 'site-packages/pip' in file:
+            continue
+
+        # file is /opt/az/lib/python3.10/site-packages/websocket/__pycache__/_app.cpython-310.pyc
+        # py_filename is _app.py
+        py_filename = Path(file).name[:-len(pyc_suffix)] + 'py'
+        # py_path is /opt/az/lib/python3.10/site-packages/websocket/_app.py
+        py_path = Path(file).parent.parent / py_filename
+        if py_path.exists():
+            py_path.unlink()
+            shutil.move(file, py_path.with_suffix('.pyc'))
+
+    for f in glob.glob(f'{folder}/**/__pycache__', recursive=True):
+        # Remove pip __pycache__ folder for Windows only to save more space
+        if 'site-packages/pip' in f and not platform.system() == 'Windows':
+            continue
+        shutil.rmtree(f)
+
+    _LOGGER.info('Finish processing')
+    _print_folder_size(folder)
+
+
+if __name__ == '__main__':
+    logging.basicConfig(level=logging.DEBUG)
+    if len(sys.argv) == 2:
+        main(sys.argv[1])
+    else:
+        main(sys.argv[1], sys.argv[2])

src/azure-cli/azure/cli/command_modules/rdbms/tests/latest/test_rdbms_flexible_commands.py

@@ -33,13 +33,13 @@
     AbstractPreparer,
     SingleValueReplacer)
 from azure.core.exceptions import HttpResponseError
-from ..._client_factory import cf_mysql_flexible_private_dns_zone_suffix_operations, cf_postgres_flexible_private_dns_zone_suffix_operations
-from ...flexible_server_virtual_network import prepare_private_network, prepare_private_dns_zone, prepare_public_network, DEFAULT_VNET_ADDRESS_PREFIX, DEFAULT_SUBNET_ADDRESS_PREFIX
-from ...flexible_server_custom_postgres import DbContext as PostgresDbContext
-from ...flexible_server_custom_mysql import DbContext as MysqlDbContext
-from ...flexible_server_custom_mysql import _determine_iops
-from ..._flexible_server_util import get_mysql_list_skus_info
-from ..._util import retryable_method
+from azure.cli.command_modules.rdbms._client_factory import cf_mysql_flexible_private_dns_zone_suffix_operations, cf_postgres_flexible_private_dns_zone_suffix_operations
+from azure.cli.command_modules.rdbms.flexible_server_virtual_network import prepare_private_network, prepare_private_dns_zone, prepare_public_network, DEFAULT_VNET_ADDRESS_PREFIX, DEFAULT_SUBNET_ADDRESS_PREFIX
+from azure.cli.command_modules.rdbms.flexible_server_custom_postgres import DbContext as PostgresDbContext
+from azure.cli.command_modules.rdbms.flexible_server_custom_mysql import DbContext as MysqlDbContext
+from azure.cli.command_modules.rdbms.flexible_server_custom_mysql import _determine_iops
+from azure.cli.command_modules.rdbms._flexible_server_util import get_mysql_list_skus_info
+from azure.cli.command_modules.rdbms._util import retryable_method
 # Constants
 SERVER_NAME_PREFIX = 'azuredbclitest-'
 SERVER_NAME_MAX_LENGTH = 20