Merge branch 'feature/optimize-CI-framework' into 'main'

ci: optimize ci configuration script

See merge request espressif/esp-zigbee-sdk!155

.gitlab-ci.yml

@@ -2,3 +2,4 @@ include:
   - '.gitlab/ci/common.yml'
   - '.gitlab/ci/generate_build_test_jobs.yml'
   - '.gitlab/ci/docs.yml'
+  - '.gitlab/ci/build_idf_examples.yml'

.gitlab/ci/build_idf_examples.yml

@@ -0,0 +1,24 @@
+build_idf_examples:
+  stage: build
+  image: ${CI_DOCKER_REGISTRY}/esp-env-${DEFAULT_BUILD_DOCKER_ENV}
+  extends:
+    - .update_zigbee_lib_script
+  artifacts:
+    paths:
+      - "**/build*/*.bin"
+      - "**/build*/*.map"
+      - "**/build*/*.elf"
+      - "**/build*/flasher_args.json"
+      - "**/build*/flash_project_args"
+      - "**/build*/config/sdkconfig.json"
+      - "**/build*/bootloader/*.bin"
+      - "**/build*/partition_table/*.bin"
+    when: always
+    expire_in: 4 days
+  variables:
+    IDF_VERSION: ${DEFAULT_IDF_VERSION}
+  script:
+      - cd ${IDF_PATH}
+      - python ${CI_PROJECT_DIR}/tools/ci/build_apps.py ./examples/zigbee
+  tags:
+    - build

.gitlab/ci/common.yml

@@ -8,6 +8,7 @@ variables:
     GIT_STRATEGY: fetch
     GIT_SUBMODULE_STRATEGY: none
     DEFAULT_IDF_VERSION: "v5.3.2"
+    DEFAULT_BUILD_DOCKER_ENV: "v5.3:1"
     DEFAULT_DOC_DOCKER_ENV: "v5.3:1-1"
     # tell build system do not check submodule update as we download archive instead of clone
     IDF_SKIP_CHECK_SUBMODULES: 1
@@ -28,7 +29,6 @@ variables:
   - git submodule update --init --depth=1
   - ./install.sh
   - . ./export.sh
-  - export PYTHONPATH="$CI_PROJECT_DIR/esp-idf/tools/ci/python_packages:$PYTHONPATH"
 
 .setup_esp_zboss_lib: &setup_esp_zboss_lib
   - cd $CI_PROJECT_DIR

.gitlab/ci/generate_build_test_jobs.yml

@@ -7,7 +7,7 @@ generate_child_pipeline:
     expire_in: 1 week
     when: always
   script:
-    - python3 tools/ci/dynamic_pipelines/generate_child_pipeline.py -p $CI_PROJECT_DIR -t .gitlab/ci/templates.yml -d ${DEFAULT_IDF_VERSION}
+    - python3 tools/ci/dynamic_pipelines/generate_child_pipeline.py -p $CI_PROJECT_DIR -t .gitlab/ci/templates.yml -i ${DEFAULT_IDF_VERSION} -d ${DEFAULT_BUILD_DOCKER_ENV}
 
 build_and_test_pipeline_default:
   stage: build

.gitlab/ci/templates.yml

@@ -59,7 +59,6 @@ include:
       - "**/build*/partition_table/*.bin"
     when: always
     expire_in: 4 days
-  parallel: 2
   script:
     - python tools/ci/build_apps.py ./examples --no_pytest
       --parallel-count ${CI_NODE_TOTAL:-1}

tools/ci/build_apps.py

@@ -7,19 +7,12 @@
 import argparse
 import sys
 import os
-import logging
-from pathlib import Path
 from typing import List
-
-idf_build_apps_logger = logging.getLogger('idf_build_apps')
+from pathlib import Path
 from idf_build_apps import App, build_apps, find_apps, setup_logging
 
 # from idf_ci_utils import IDF_PATH, get_pytest_app_paths, get_pytest_cases, get_ttfw_app_paths
-
-PROJECT_ROOT = Path(__file__).parent.parent.parent.absolute()
-DEF_APP_PATH = PROJECT_ROOT / 'examples'
 APPS_BUILD_PER_JOB = 30
-
 PYTEST_APPS = [
     {"target": "esp32h2", "name": "esp_zigbee_all_device_types_app"},
     {"target": "esp32h2", "name": "HA_color_dimmable_light"},
@@ -42,18 +35,13 @@
     {"target": "esp32c6", "name": "touchlink_light"},
 ]
 
-GATEWAY_APPS = [{"target": "esp32h2", "name": "esp_zigbee_all_device_types_app"},]
-
+GATEWAY_APPS = [{"target": "esp32h2", "name": "esp_zigbee_all_device_types_app"}, ]
 
 IGNORE_WARNINGS = [
     r"warning: 'init_spiffs' defined but not used",
     r"warning: 'esp_zb_gateway_board_try_update' defined but not used",
     r"DeprecationWarning: pkg_resources is deprecated as an API",
 ]
-MAINFEST_FILES = [
-    str(PROJECT_ROOT / 'examples' / '.build-rules.yml'),
-]
-
 
 def _is_pytest_app(app: App, app_list) -> bool:
     for pytest_app in app_list:
@@ -63,26 +51,28 @@ def _is_pytest_app(app: App, app_list) -> bool:
 
 
 def get_cmake_apps(
-        paths: List[str],
+        path: str,
         target: str,
         config_rules_str: List[str],
 ) -> List[App]:
+    current_working_dir = Path.cwd()
+    manifest_files = str(current_working_dir / path / '.build-test-rules.yml')
+    print('manifest path:{}'.format(manifest_files))
+    if not os.path.exists(manifest_files):
+        raise FileNotFoundError(f"The file '{manifest_files}' does not exist.")
     apps = find_apps(
-        paths,
+        path,
         recursive=True,
         target=target,
         build_dir='build_@t_@w',
         config_rules_str=config_rules_str,
         check_warnings=True,
-        manifest_files=MAINFEST_FILES,
+        manifest_files=manifest_files,
     )
     return apps
 
 def main(args: argparse.Namespace) -> None:
-    current_dir = os.getcwd()
-    os.chdir(current_dir)
-    apps = get_cmake_apps(args.paths, args.target, args.config)
-
+    apps = get_cmake_apps(args.path, args.target, args.config)
     # no_pytest and only_pytest can not be both True
     assert not (args.no_pytest and args.pytest)
     if args.no_pytest:
@@ -94,12 +84,10 @@ def main(args: argparse.Namespace) -> None:
         apps_for_build = [app for app in apps if _is_pytest_app(app, GATEWAY_APPS)]
     else:
         apps_for_build = apps[:]
+    assert apps_for_build, 'Found no apps for build'
+    print('Found %d apps after filtering', len(apps_for_build))
+    print('Suggest setting the parallel count to %d for this build job', len(apps_for_build) // APPS_BUILD_PER_JOB + 1)
 
-    logging.info('Found %d apps after filtering', len(apps_for_build))
-    logging.info(
-        'Suggest setting the parallel count to %d for this build job',
-        len(apps_for_build) // APPS_BUILD_PER_JOB + 1,
-    )
     ignore_warnings = IGNORE_WARNINGS
     if args.ignore_warning:
         ignore_warnings = [r"warning: .*"]
@@ -124,7 +112,7 @@ def main(args: argparse.Namespace) -> None:
         description='Build all the apps for different test types. Will auto remove those non-test apps binaries',
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
-    parser.add_argument('paths', nargs='*', help='Paths to the apps to build.')
+    parser.add_argument('path', help='Path to the file .build-test-rules.yml, it must be a relative path')
     parser.add_argument(
         '-t',
         '--target',
@@ -171,15 +159,11 @@ def main(args: argparse.Namespace) -> None:
         action="store_true",
         help='Only build rcp_gateway pytest apps, defined in GATEWAY_APPS',
     )
-
     parser.add_argument(
         '--ignore_warning',
         action="store_true",
         help='Ignore all warnings',
     )
-
     arguments = parser.parse_args()
-    if not arguments.paths:
-        arguments.paths = [DEF_APP_PATH]
     setup_logging(verbose=1)  # Info
     main(arguments)

tools/ci/dynamic_pipelines/constants.py

@@ -3,11 +3,10 @@
 
 
 class ZbCiCons:
-    IDF_DOCKER_ENV = {
-        'v5.1.4':'v5.1:1',
-        'v5.2.3':'v5.2:2',
-        'v5.3.2':'v5.3:1',
+    LTS_IDF_DOCKER = {
+        'v5.1.4': 'v5.1:1',
+        'v5.2.3': 'v5.2:2',
     }
     CHIPS = ['esp32c6', 'esp32h2', 'esp32s3']
     PIPELINE_YAML_DEFAULT = 'child_pipeline_default.yml'
-    PIPELINE_YAML_FULL = 'child_pipeline_lts.yml'
+    PIPELINE_YAML_LTS = 'child_pipeline_lts.yml'

tools/ci/dynamic_pipelines/generate_child_pipeline.py

@@ -52,14 +52,16 @@ def main(arg):
     project_path = arg.project_path
     assert project_path
     template_yaml = arg.template_path
+    if not os.path.exists(template_yaml):
+        raise FileNotFoundError('The file {} does not exist.'.format(template_yaml))
     default_idf = arg.default_idf_version
-    idf_and_docker_lts = ZbCiCons.IDF_DOCKER_ENV.copy()
-    del idf_and_docker_lts[default_idf]
-    idf_and_docker_default = {default_idf:ZbCiCons.IDF_DOCKER_ENV.get(default_idf)}
+    default_docker_version = arg.default_docker_version
+    lts_idf_docker = ZbCiCons.LTS_IDF_DOCKER
     default_yaml_path = os.path.join(project_path, ZbCiCons.PIPELINE_YAML_DEFAULT)
-    full_yaml_path = os.path.join(project_path, ZbCiCons.PIPELINE_YAML_FULL)
-    generate(idf_and_docker_default, default_yaml_path, template_yaml)
-    generate(idf_and_docker_lts, full_yaml_path, template_yaml)
+    lts_yaml_path = os.path.join(project_path, ZbCiCons.PIPELINE_YAML_LTS)
+    default_idf_docker = {default_idf: default_docker_version}
+    generate(default_idf_docker, default_yaml_path, template_yaml)
+    generate(lts_idf_docker, lts_yaml_path, template_yaml)
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(
@@ -69,20 +71,26 @@ def main(arg):
     parser.add_argument(
         '-p',
         '--project_path',
-        default=None,
+        required=True,
         help='Paths to project',
     )
     parser.add_argument(
         '-t',
         '--template_path',
-        default=None,
+        required=True,
         help='Paths to template.yml',
     )
     parser.add_argument(
-        '-d',
+        '-i',
         '--default_idf_version',
-        default=None,
+        required=True,
         help='default_idf_version',
     )
+    parser.add_argument(
+        '-d',
+        '--default_docker_version',
+        required=True,
+        help='default_docker_version',
+    )
     args = parser.parse_args()
     main(args)

tools/ci/update_cmake_files.sh

@@ -17,6 +17,8 @@ update_dependency(){
     sed -i "/\besp-zboss-lib\b/d;/\besp-zigbee-lib\b/d" "${ESP_ZIGBEE_SDK}"/examples/**/main/idf_component.yml
     #   for components in ESP-ZIGBEE-SDK
     sed -i "/\besp-zboss-lib\b/d;/\besp-zigbee-lib\b/d" "${ESP_ZIGBEE_SDK}"/components/*/idf_component.yml
+    #   for examples in idf examples
+    sed -i "/\besp-zboss-lib\b/d;/\besp-zigbee-lib\b/d" "${IDF_PATH}"/examples/zigbee/**/main/idf_component.yml
 
     # Link the local LIB components to the components directory of ESP-IDF
     for component in "${!LIB_COMPONENTS[@]}"; do