Merge branch 'ci/fix_artifacts_object_name' into 'master'

ci: fix job name sanitizer

See merge request espressif/esp-idf!28676

tools/ci/artifacts_handler.py

@@ -1,17 +1,17 @@
-# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 import argparse
 import fnmatch
 import glob
 import os
-import re
 import typing as t
 import zipfile
 from enum import Enum
 from pathlib import Path
 from zipfile import ZipFile
 
 import urllib3
+from idf_ci_utils import sanitize_job_name
 from idf_pytest.constants import DEFAULT_BUILD_LOG_FILENAME
 from minio import Minio
 
@@ -149,8 +149,7 @@ def _upload_files(
 
     try:
         if has_file:
-            job_name_sanitized = re.sub(r' \[\d+]', '', job_name)
-            obj_name = f'{pipeline_id}/{artifact_type.value}/{job_name_sanitized}/{job_id}.zip'
+            obj_name = f'{pipeline_id}/{artifact_type.value}/{sanitize_job_name(job_name)}/{job_id}.zip'
             print(f'Created archive file: {job_id}.zip, uploading as {obj_name}')
 
             client.fput_object(getenv('IDF_S3_BUCKET'), obj_name, f'{job_id}.zip')

tools/ci/idf_ci_utils.py

@@ -1,11 +1,10 @@
-# SPDX-FileCopyrightText: 2020-2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2020-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-
 # internal use only for CI
 # some CI related util functions
-
 import logging
 import os
+import re
 import subprocess
 import sys
 import typing as t
@@ -178,10 +177,6 @@ def _is_rule_key(key: str) -> bool:
 
 
 def get_all_manifest_files() -> t.List[str]:
-    """
-
-    :rtype: object
-    """
     paths: t.List[str] = []
 
     for p in Path(IDF_PATH).glob('**/.build-test-rules.yml'):
@@ -191,3 +186,20 @@ def get_all_manifest_files() -> t.List[str]:
         paths.append(str(p))
 
     return paths
+
+
+def sanitize_job_name(name: str) -> str:
+    """
+    Sanitize the job name from CI_JOB_NAME
+
+    - for job with `parallel: int` set, the `CI_JOB_NAME` would be `job_name index/total`, like `foo 1/3`
+    - for job with `parallel: matrix` set, the `CI_JOB_NAME` would be `job_name: [var1, var2]`, like `foo: [a, b]`
+
+    We consider
+    - the jobs generated by `parallel: int` as the same job, i.e., we remove the index/total part.
+    - the jobs generated by `parallel: matrix` as different jobs, so we keep the matrix part.
+
+    :param name: job name
+    :return: sanitized job name
+    """
+    return re.sub(r' \d+/\d+', '', name)