Skip to content

Commit

Permalink
Merge branch 'ci/move_stdout_print_to_stderr_logging' into 'master'
Browse files Browse the repository at this point in the history
ci: move stdout print to stderr logging in gitlab api

See merge request espressif/esp-idf!22387
  • Loading branch information
hfudev committed Feb 16, 2023
2 parents 76989b5 + 8d0dda2 commit 610102a
Showing 1 changed file with 18 additions and 9 deletions.
27 changes: 18 additions & 9 deletions tools/ci/python_packages/gitlab_api.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import argparse
import logging
import os
import re
import tarfile
Expand All @@ -14,12 +15,15 @@

TR = Callable[..., Any]

logging.basicConfig(level=logging.INFO)


def retry(func: TR) -> TR:
"""
This wrapper will only catch several exception types associated with
"network issues" and retry the whole function.
"""

@wraps(func)
def wrapper(self: 'Gitlab', *args: Any, **kwargs: Any) -> Any:
retried = 0
Expand All @@ -41,12 +45,15 @@ def wrapper(self: 'Gitlab', *args: Any, **kwargs: Any) -> Any:
if retried > self.DOWNLOAD_ERROR_MAX_RETRIES:
raise e # get out of the loop
else:
print('Network failure in {}, retrying ({})'.format(getattr(func, '__name__', '(unknown callable)'), retried))
logging.warning(
'Network failure in {}, retrying ({})'.format(getattr(func, '__name__', '(unknown callable)'),
retried))
time.sleep(2 ** retried) # wait a bit more after each retry
continue
else:
break
return res

return wrapper


Expand Down Expand Up @@ -138,7 +145,7 @@ def download_artifact(self, job_id: int, artifact_path: str, destination: Option
try:
data = job.artifact(a_path) # type: bytes
except gitlab.GitlabGetError as e:
print("Failed to download '{}' from job {}".format(a_path, job_id))
logging.error("Failed to download '{}' from job {}".format(a_path, job_id))
raise e
raw_data_list.append(data)
if destination:
Expand Down Expand Up @@ -177,7 +184,8 @@ def find_job_id(self, job_name: str, pipeline_id: Optional[str] = None, job_stat
return job_id_list

@retry
def download_archive(self, ref: str, destination: str, project_id: Optional[int] = None, cache_dir: Optional[str] = None) -> str:
def download_archive(self, ref: str, destination: str, project_id: Optional[int] = None,
cache_dir: Optional[str] = None) -> str:
"""
Download archive of certain commit of a repository and extract to destination path
Expand All @@ -195,15 +203,16 @@ def download_archive(self, ref: str, destination: str, project_id: Optional[int]
local_archive_file = os.path.join(cache_dir, f'{ref}.tar.gz')
os.makedirs(os.path.dirname(local_archive_file), exist_ok=True)
if os.path.isfile(local_archive_file):
print('Use cached archive file. Skipping download...')
logging.info('Use cached archive file. Skipping download...')
else:
with open(local_archive_file, 'wb') as fw:
try:
project.repository_archive(sha=ref, streamed=True, action=fw.write)
except gitlab.GitlabGetError as e:
print('Failed to archive from project {}'.format(project_id))
logging.error('Failed to archive from project {}'.format(project_id))
raise e
print('Downloaded archive size: {:.03f}MB'.format(float(os.path.getsize(local_archive_file)) / (1024 * 1024)))
logging.info('Downloaded archive size: {:.03f}MB'.format(
float(os.path.getsize(local_archive_file)) / (1024 * 1024)))

return self.decompress_archive(local_archive_file, destination)

Expand All @@ -212,10 +221,10 @@ def download_archive(self, ref: str, destination: str, project_id: Optional[int]
try:
project.repository_archive(sha=ref, streamed=True, action=temp_file.write)
except gitlab.GitlabGetError as e:
print('Failed to archive from project {}'.format(project_id))
logging.error('Failed to archive from project {}'.format(project_id))
raise e

print('Downloaded archive size: {:.03f}MB'.format(float(os.path.getsize(temp_file.name)) / (1024 * 1024)))
logging.info('Downloaded archive size: {:.03f}MB'.format(float(os.path.getsize(temp_file.name)) / (1024 * 1024)))

return self.decompress_archive(temp_file.name, destination)

Expand Down

0 comments on commit 610102a

Please sign in to comment.