From 3947d638732973c7d372a492bdd429c52c5a8221 Mon Sep 17 00:00:00 2001 From: liuqiang Date: Wed, 11 Dec 2024 16:08:23 +0800 Subject: [PATCH] ci: optimize ci configuration script --- .gitlab-ci.yml | 1 + .gitlab/ci/build_idf_examples.yml | 24 ++++++++++ .gitlab/ci/common.yml | 2 +- .gitlab/ci/generate_build_test_jobs.yml | 2 +- .gitlab/ci/templates.yml | 1 - ....build-rules.yml => .build-test-rules.yml} | 0 tools/ci/build_apps.py | 46 ++++++------------- tools/ci/dynamic_pipelines/constants.py | 9 ++-- .../generate_child_pipeline.py | 28 +++++++---- tools/ci/update_cmake_files.sh | 2 + 10 files changed, 66 insertions(+), 49 deletions(-) create mode 100644 .gitlab/ci/build_idf_examples.yml rename examples/{.build-rules.yml => .build-test-rules.yml} (100%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ed173dd..328117b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,3 +2,4 @@ include: - '.gitlab/ci/common.yml' - '.gitlab/ci/generate_build_test_jobs.yml' - '.gitlab/ci/docs.yml' + - '.gitlab/ci/build_idf_examples.yml' diff --git a/.gitlab/ci/build_idf_examples.yml b/.gitlab/ci/build_idf_examples.yml new file mode 100644 index 0000000..0733f9d --- /dev/null +++ b/.gitlab/ci/build_idf_examples.yml @@ -0,0 +1,24 @@ +build_idf_examples: + stage: build + image: ${CI_DOCKER_REGISTRY}/esp-env-${DEFAULT_BUILD_DOCKER_ENV} + extends: + - .update_zigbee_lib_script + artifacts: + paths: + - "**/build*/*.bin" + - "**/build*/*.map" + - "**/build*/*.elf" + - "**/build*/flasher_args.json" + - "**/build*/flash_project_args" + - "**/build*/config/sdkconfig.json" + - "**/build*/bootloader/*.bin" + - "**/build*/partition_table/*.bin" + when: always + expire_in: 4 days + variables: + IDF_VERSION: ${DEFAULT_IDF_VERSION} + script: + - cd ${IDF_PATH} + - python ${CI_PROJECT_DIR}/tools/ci/build_apps.py ./examples/zigbee + tags: + - build diff --git a/.gitlab/ci/common.yml b/.gitlab/ci/common.yml index abf6bfd..15ae304 100644 --- a/.gitlab/ci/common.yml +++ b/.gitlab/ci/common.yml @@ -8,6 +8,7 @@ variables: GIT_STRATEGY: fetch GIT_SUBMODULE_STRATEGY: none DEFAULT_IDF_VERSION: "v5.3.2" + DEFAULT_BUILD_DOCKER_ENV: "v5.3:1" DEFAULT_DOC_DOCKER_ENV: "v5.3:1-1" # tell build system do not check submodule update as we download archive instead of clone IDF_SKIP_CHECK_SUBMODULES: 1 @@ -28,7 +29,6 @@ variables: - git submodule update --init --depth=1 - ./install.sh - . ./export.sh - - export PYTHONPATH="$CI_PROJECT_DIR/esp-idf/tools/ci/python_packages:$PYTHONPATH" .setup_esp_zboss_lib: &setup_esp_zboss_lib - cd $CI_PROJECT_DIR diff --git a/.gitlab/ci/generate_build_test_jobs.yml b/.gitlab/ci/generate_build_test_jobs.yml index 572565d..5baa64c 100644 --- a/.gitlab/ci/generate_build_test_jobs.yml +++ b/.gitlab/ci/generate_build_test_jobs.yml @@ -7,7 +7,7 @@ generate_child_pipeline: expire_in: 1 week when: always script: - - python3 tools/ci/dynamic_pipelines/generate_child_pipeline.py -p $CI_PROJECT_DIR -t .gitlab/ci/templates.yml -d ${DEFAULT_IDF_VERSION} + - python3 tools/ci/dynamic_pipelines/generate_child_pipeline.py -p $CI_PROJECT_DIR -t .gitlab/ci/templates.yml -i ${DEFAULT_IDF_VERSION} -d ${DEFAULT_BUILD_DOCKER_ENV} build_and_test_pipeline_default: stage: build diff --git a/.gitlab/ci/templates.yml b/.gitlab/ci/templates.yml index 4fbbdf8..25d084d 100644 --- a/.gitlab/ci/templates.yml +++ b/.gitlab/ci/templates.yml @@ -59,7 +59,6 @@ include: - "**/build*/partition_table/*.bin" when: always expire_in: 4 days - parallel: 2 script: - python tools/ci/build_apps.py ./examples --no_pytest --parallel-count ${CI_NODE_TOTAL:-1} diff --git a/examples/.build-rules.yml b/examples/.build-test-rules.yml similarity index 100% rename from examples/.build-rules.yml rename to examples/.build-test-rules.yml diff --git a/tools/ci/build_apps.py b/tools/ci/build_apps.py index c169ecc..4c8f816 100644 --- a/tools/ci/build_apps.py +++ b/tools/ci/build_apps.py @@ -7,19 +7,12 @@ import argparse import sys import os -import logging -from pathlib import Path from typing import List - -idf_build_apps_logger = logging.getLogger('idf_build_apps') +from pathlib import Path from idf_build_apps import App, build_apps, find_apps, setup_logging # from idf_ci_utils import IDF_PATH, get_pytest_app_paths, get_pytest_cases, get_ttfw_app_paths - -PROJECT_ROOT = Path(__file__).parent.parent.parent.absolute() -DEF_APP_PATH = PROJECT_ROOT / 'examples' APPS_BUILD_PER_JOB = 30 - PYTEST_APPS = [ {"target": "esp32h2", "name": "esp_zigbee_all_device_types_app"}, {"target": "esp32h2", "name": "HA_color_dimmable_light"}, @@ -42,18 +35,13 @@ {"target": "esp32c6", "name": "touchlink_light"}, ] -GATEWAY_APPS = [{"target": "esp32h2", "name": "esp_zigbee_all_device_types_app"},] - +GATEWAY_APPS = [{"target": "esp32h2", "name": "esp_zigbee_all_device_types_app"}, ] IGNORE_WARNINGS = [ r"warning: 'init_spiffs' defined but not used", r"warning: 'esp_zb_gateway_board_try_update' defined but not used", r"DeprecationWarning: pkg_resources is deprecated as an API", ] -MAINFEST_FILES = [ - str(PROJECT_ROOT / 'examples' / '.build-rules.yml'), -] - def _is_pytest_app(app: App, app_list) -> bool: for pytest_app in app_list: @@ -63,26 +51,28 @@ def _is_pytest_app(app: App, app_list) -> bool: def get_cmake_apps( - paths: List[str], + path: str, target: str, config_rules_str: List[str], ) -> List[App]: + current_working_dir = Path.cwd() + manifest_files = str(current_working_dir / path / '.build-test-rules.yml') + print('manifest path:{}'.format(manifest_files)) + if not os.path.exists(manifest_files): + raise FileNotFoundError(f"The file '{manifest_files}' does not exist.") apps = find_apps( - paths, + path, recursive=True, target=target, build_dir='build_@t_@w', config_rules_str=config_rules_str, check_warnings=True, - manifest_files=MAINFEST_FILES, + manifest_files=manifest_files, ) return apps def main(args: argparse.Namespace) -> None: - current_dir = os.getcwd() - os.chdir(current_dir) - apps = get_cmake_apps(args.paths, args.target, args.config) - + apps = get_cmake_apps(args.path, args.target, args.config) # no_pytest and only_pytest can not be both True assert not (args.no_pytest and args.pytest) if args.no_pytest: @@ -94,12 +84,10 @@ def main(args: argparse.Namespace) -> None: apps_for_build = [app for app in apps if _is_pytest_app(app, GATEWAY_APPS)] else: apps_for_build = apps[:] + assert apps_for_build, 'Found no apps for build' + print('Found %d apps after filtering', len(apps_for_build)) + print('Suggest setting the parallel count to %d for this build job', len(apps_for_build) // APPS_BUILD_PER_JOB + 1) - logging.info('Found %d apps after filtering', len(apps_for_build)) - logging.info( - 'Suggest setting the parallel count to %d for this build job', - len(apps_for_build) // APPS_BUILD_PER_JOB + 1, - ) ignore_warnings = IGNORE_WARNINGS if args.ignore_warning: ignore_warnings = [r"warning: .*"] @@ -124,7 +112,7 @@ def main(args: argparse.Namespace) -> None: description='Build all the apps for different test types. Will auto remove those non-test apps binaries', formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - parser.add_argument('paths', nargs='*', help='Paths to the apps to build.') + parser.add_argument('path', help='Path to the file .build-test-rules.yml, it must be a relative path') parser.add_argument( '-t', '--target', @@ -171,15 +159,11 @@ def main(args: argparse.Namespace) -> None: action="store_true", help='Only build rcp_gateway pytest apps, defined in GATEWAY_APPS', ) - parser.add_argument( '--ignore_warning', action="store_true", help='Ignore all warnings', ) - arguments = parser.parse_args() - if not arguments.paths: - arguments.paths = [DEF_APP_PATH] setup_logging(verbose=1) # Info main(arguments) diff --git a/tools/ci/dynamic_pipelines/constants.py b/tools/ci/dynamic_pipelines/constants.py index 51dfc6d..79b969c 100644 --- a/tools/ci/dynamic_pipelines/constants.py +++ b/tools/ci/dynamic_pipelines/constants.py @@ -3,11 +3,10 @@ class ZbCiCons: - IDF_DOCKER_ENV = { - 'v5.1.4':'v5.1:1', - 'v5.2.3':'v5.2:2', - 'v5.3.2':'v5.3:1', + LTS_IDF_DOCKER = { + 'v5.1.4': 'v5.1:1', + 'v5.2.3': 'v5.2:2', } CHIPS = ['esp32c6', 'esp32h2', 'esp32s3'] PIPELINE_YAML_DEFAULT = 'child_pipeline_default.yml' - PIPELINE_YAML_FULL = 'child_pipeline_lts.yml' + PIPELINE_YAML_LTS = 'child_pipeline_lts.yml' diff --git a/tools/ci/dynamic_pipelines/generate_child_pipeline.py b/tools/ci/dynamic_pipelines/generate_child_pipeline.py index 5a3e0a7..362963b 100644 --- a/tools/ci/dynamic_pipelines/generate_child_pipeline.py +++ b/tools/ci/dynamic_pipelines/generate_child_pipeline.py @@ -52,14 +52,16 @@ def main(arg): project_path = arg.project_path assert project_path template_yaml = arg.template_path + if not os.path.exists(template_yaml): + raise FileNotFoundError('The file {} does not exist.'.format(template_yaml)) default_idf = arg.default_idf_version - idf_and_docker_lts = ZbCiCons.IDF_DOCKER_ENV.copy() - del idf_and_docker_lts[default_idf] - idf_and_docker_default = {default_idf:ZbCiCons.IDF_DOCKER_ENV.get(default_idf)} + default_docker_version = arg.default_docker_version + lts_idf_docker = ZbCiCons.LTS_IDF_DOCKER default_yaml_path = os.path.join(project_path, ZbCiCons.PIPELINE_YAML_DEFAULT) - full_yaml_path = os.path.join(project_path, ZbCiCons.PIPELINE_YAML_FULL) - generate(idf_and_docker_default, default_yaml_path, template_yaml) - generate(idf_and_docker_lts, full_yaml_path, template_yaml) + lts_yaml_path = os.path.join(project_path, ZbCiCons.PIPELINE_YAML_LTS) + default_idf_docker = {default_idf: default_docker_version} + generate(default_idf_docker, default_yaml_path, template_yaml) + generate(lts_idf_docker, lts_yaml_path, template_yaml) if __name__ == '__main__': parser = argparse.ArgumentParser( @@ -69,20 +71,26 @@ def main(arg): parser.add_argument( '-p', '--project_path', - default=None, + required=True, help='Paths to project', ) parser.add_argument( '-t', '--template_path', - default=None, + required=True, help='Paths to template.yml', ) parser.add_argument( - '-d', + '-i', '--default_idf_version', - default=None, + required=True, help='default_idf_version', ) + parser.add_argument( + '-d', + '--default_docker_version', + required=True, + help='default_docker_version', + ) args = parser.parse_args() main(args) diff --git a/tools/ci/update_cmake_files.sh b/tools/ci/update_cmake_files.sh index 9f75753..42407ab 100755 --- a/tools/ci/update_cmake_files.sh +++ b/tools/ci/update_cmake_files.sh @@ -17,6 +17,8 @@ update_dependency(){ sed -i "/\besp-zboss-lib\b/d;/\besp-zigbee-lib\b/d" "${ESP_ZIGBEE_SDK}"/examples/**/main/idf_component.yml # for components in ESP-ZIGBEE-SDK sed -i "/\besp-zboss-lib\b/d;/\besp-zigbee-lib\b/d" "${ESP_ZIGBEE_SDK}"/components/*/idf_component.yml + # for examples in idf examples + sed -i "/\besp-zboss-lib\b/d;/\besp-zigbee-lib\b/d" "${IDF_PATH}"/examples/zigbee/**/main/idf_component.yml # Link the local LIB components to the components directory of ESP-IDF for component in "${!LIB_COMPONENTS[@]}"; do