From 92742e14bc49106495dfa57968dccd7f567cba60 Mon Sep 17 00:00:00 2001 From: leoecrepont Date: Wed, 20 Nov 2024 15:38:03 +0100 Subject: [PATCH] chore(ci-issues): add junit upload CLI Adds the cli `mergify ci-issues junit-upload` to upload JUnit xml reports to CI Issues. Fixes MRGFY-4339 --- mergify_cli/ci/__init__.py | 0 mergify_cli/ci/cli.py | 131 ++++++++++++++++++ mergify_cli/ci/junit_upload.py | 76 ++++++++++ mergify_cli/cli.py | 2 + mergify_cli/tests/ci_issues/__init__.py | 0 .../tests/ci_issues/reports/report.xml | 15 ++ .../tests/ci_issues/test_junit_upload.py | 119 ++++++++++++++++ mergify_cli/utils.py | 37 ++++- 8 files changed, 376 insertions(+), 4 deletions(-) create mode 100644 mergify_cli/ci/__init__.py create mode 100644 mergify_cli/ci/cli.py create mode 100644 mergify_cli/ci/junit_upload.py create mode 100644 mergify_cli/tests/ci_issues/__init__.py create mode 100644 mergify_cli/tests/ci_issues/reports/report.xml create mode 100644 mergify_cli/tests/ci_issues/test_junit_upload.py diff --git a/mergify_cli/ci/__init__.py b/mergify_cli/ci/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mergify_cli/ci/cli.py b/mergify_cli/ci/cli.py new file mode 100644 index 0000000..fbfca37 --- /dev/null +++ b/mergify_cli/ci/cli.py @@ -0,0 +1,131 @@ +import os +import re +import typing + +import click + +from mergify_cli import console +from mergify_cli import utils +from mergify_cli.ci import junit_upload as junit_upload_mod + + +ci = click.Group( + "ci", + help="Mergify's CI related commands", +) + + +def get_ci_provider() -> typing.Literal["github_action", "circleci"] | None: + if os.getenv("GITHUB_ACTIONS") == "true": + return "github_action" + if os.getenv("CIRCLECI") == "true": + return "circleci" + return None + + +def get_job_name() -> str | None: + if os.getenv("GITHUB_ACTIONS") == "true": + return os.getenv("GITHUB_WORKFLOW") + if os.getenv("CIRCLECI") == "true": + return os.getenv("CIRCLE_JOB") + + console.log("Error: failed to get the job's name from env", style="red") + return None + + +def get_head_sha() -> str | None: + if os.getenv("GITHUB_ACTIONS") == "true": + return os.getenv("GITHUB_SHA") + if os.getenv("CIRCLECI") == "true": + return os.getenv("CIRCLE_SHA1") + + console.log("Error: failed to get the head SHA from env", style="red") + return None + + +def get_github_repository() -> str | None: + if os.getenv("GITHUB_ACTIONS") == "true": + return os.getenv("GITHUB_REPOSITORY") + if os.getenv("CIRCLECI") == "true": + repository_url = os.getenv("CIRCLE_REPOSITORY_URL") + if repository_url and ( + match := re.match( + r"(https?://[\w.-]+/)?(?P[\w.-]+/[\w.-]+)/?$", + repository_url, + ) + ): + return match.group("full_name") + + console.log("Error: failed to get the GitHub repository from env", style="red") + return None + + +@ci.command(help="Upload JUnit XML reports") +@click.option( + "--api-url", + "-u", + help="URL of the Mergify API", + required=True, + envvar="MERGIFY_API_SERVER", + default="https://api.mergify.com/", + show_default=True, +) +@click.option( + "--token", + "-t", + help="CI Issues Application Key", + required=True, + envvar="MERGIFY_CI_ISSUES_TOKEN", +) +@click.option( + "--repository", + "-r", + help="Repository full name (owner/repo)", + required=True, + default=get_github_repository, +) +@click.option( + "--head-sha", + "-s", + help="Head SHA of the triggered job", + required=True, + default=get_head_sha, +) +@click.option( + "--job-name", + "-j", + help="Job's name", + required=True, + default=get_job_name, +) +@click.option( + "--provider", + "-p", + help="CI provider", + default=get_ci_provider, +) +@click.argument( + "files", + nargs=-1, + required=True, + type=click.Path(exists=True, dir_okay=False), +) +@utils.run_with_asyncio +async def junit_upload( # noqa: PLR0913, PLR0917 + api_url: str, + token: str, + repository: str, + head_sha: str, + job_name: str, + provider: str | None, + files: tuple[str, ...], +) -> None: + await junit_upload_mod.upload( + api_url=api_url, + token=token, + repository=repository, + head_sha=head_sha, + job_name=job_name, + provider=provider, + files=files, + ) diff --git a/mergify_cli/ci/junit_upload.py b/mergify_cli/ci/junit_upload.py new file mode 100644 index 0000000..a078c9e --- /dev/null +++ b/mergify_cli/ci/junit_upload.py @@ -0,0 +1,76 @@ +import pathlib +import typing + +import httpx + +from mergify_cli import console +from mergify_cli import utils + + +def get_files_to_upload( + files: tuple[str, ...], +) -> list[tuple[str, tuple[str, typing.BinaryIO, str]]]: + files_to_upload: list[tuple[str, tuple[str, typing.BinaryIO, str]]] = [] + + for file in set(files): + file_path = pathlib.Path(file) + files_to_upload.append( + ("files", (file_path.name, file_path.open("rb"), "application/xml")), + ) + + return files_to_upload + + +async def raise_for_status(response: httpx.Response) -> None: + if response.is_error: + await response.aread() + details = response.text or "" + console.log(f"[red]Error details: {details}[/]") + + response.raise_for_status() + + +def get_ci_issues_client( + api_url: str, + token: str, +) -> httpx.AsyncClient: + return utils.get_http_client( + api_url, + headers={ + "Authorization": f"Bearer {token}", + }, + event_hooks={ + "request": [], + "response": [raise_for_status], + }, + ) + + +async def upload( # noqa: PLR0913, PLR0917 + api_url: str, + token: str, + repository: str, + head_sha: str, + job_name: str, + provider: str | None, + files: tuple[str, ...], +) -> None: + form_data = { + "head_sha": head_sha, + "name": job_name, + } + if provider is not None: + form_data["provider"] = provider + + files_to_upload = get_files_to_upload(files) + + async with get_ci_issues_client(api_url, token) as client: + response = await client.post( + f"v1/repos/{repository}/ci_issues_upload", + data=form_data, + files=files_to_upload, + ) + + console.log( + f"[green]:tada: File(s) uploaded (gigid={response.json()['gigid']})[/]", + ) diff --git a/mergify_cli/cli.py b/mergify_cli/cli.py index 5df9ffb..663edd2 100644 --- a/mergify_cli/cli.py +++ b/mergify_cli/cli.py @@ -26,6 +26,7 @@ from mergify_cli import VERSION from mergify_cli import console from mergify_cli import utils +from mergify_cli.ci import cli as ci_cli_mod from mergify_cli.stack import cli as stack_cli_mod @@ -91,6 +92,7 @@ def cli( cli.add_command(stack_cli_mod.stack) +cli.add_command(ci_cli_mod.ci) def main() -> None: diff --git a/mergify_cli/tests/ci_issues/__init__.py b/mergify_cli/tests/ci_issues/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mergify_cli/tests/ci_issues/reports/report.xml b/mergify_cli/tests/ci_issues/reports/report.xml new file mode 100644 index 0000000..2da49c3 --- /dev/null +++ b/mergify_cli/tests/ci_issues/reports/report.xml @@ -0,0 +1,15 @@ + + + + + + def test_failed() -> None: + > assert 1 == 0 + E assert 1 == 0 + + mergify/tests/test_junit.py:6: AssertionError + + + + \ No newline at end of file diff --git a/mergify_cli/tests/ci_issues/test_junit_upload.py b/mergify_cli/tests/ci_issues/test_junit_upload.py new file mode 100644 index 0000000..ccc5c42 --- /dev/null +++ b/mergify_cli/tests/ci_issues/test_junit_upload.py @@ -0,0 +1,119 @@ +import pathlib +from unittest import mock + +from click import testing +import httpx +import pytest +import respx + +from mergify_cli.ci import cli as cli_junit_upload +from mergify_cli.ci import junit_upload as junit_upload_mod + + +REPORT_XML = pathlib.Path(__file__).parent / "reports" / "report.xml" + + +@pytest.mark.parametrize( + ("env", "provider"), + [ + ( + { + "GITHUB_ACTIONS": "true", + "MERGIFY_API_SERVER": "https://api.mergify.com/", + "MERGIFY_CI_ISSUES_TOKEN": "abc", + "GITHUB_REPOSITORY": "user/repo", + "GITHUB_SHA": "3af96aa24f1d32fcfbb7067793cacc6dc0c6b199", + "GITHUB_WORKFLOW": "JOB", + }, + "github_action", + ), + ( + { + "CIRCLECI": "true", + "MERGIFY_API_SERVER": "https://api.mergify.com/", + "MERGIFY_CI_ISSUES_TOKEN": "abc", + "CIRCLE_REPOSITORY_URL": "https://github.com/user/repo", + "CIRCLE_SHA1": "3af96aa24f1d32fcfbb7067793cacc6dc0c6b199", + "CIRCLE_JOB": "JOB", + }, + "circleci", + ), + ], +) +def test_options_values_from_env_new( + env: dict[str, str], + provider: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + for key, value in env.items(): + monkeypatch.setenv(key, value) + + runner = testing.CliRunner() + + with mock.patch.object( + junit_upload_mod, + "upload", + mock.AsyncMock(), + ) as mocked_upload: + result = runner.invoke( + cli_junit_upload.junit_upload, + [str(REPORT_XML)], + ) + assert result.exit_code == 0 + assert mocked_upload.call_count == 1 + assert mocked_upload.call_args.kwargs == { + "provider": provider, + "api_url": "https://api.mergify.com/", + "token": "abc", + "repository": "user/repo", + "head_sha": "3af96aa24f1d32fcfbb7067793cacc6dc0c6b199", + "job_name": "JOB", + "files": (str(REPORT_XML),), + } + + +def test_get_files_to_upload() -> None: + files_to_upload = junit_upload_mod.get_files_to_upload( + (str(REPORT_XML),), + ) + assert len(files_to_upload) == 1 + assert files_to_upload[0][1][0] == "report.xml" + assert files_to_upload[0][1][1].read() == REPORT_XML.read_bytes() + assert files_to_upload[0][1][2] == "application/xml" + + +async def test_junit_upload(respx_mock: respx.MockRouter) -> None: + respx_mock.post( + "/v1/repos/user/repo/ci_issues_upload", + ).respond( + 200, + json={"gigid": "1234azertyuiop"}, + ) + + await junit_upload_mod.upload( + "https://api.mergify.com/", + "token", + "user/repo", + "3af96aa24f1d32fcfbb7067793cacc6dc0c6b199", + "ci-test-job", + "circleci", + (str(REPORT_XML),), + ) + + +async def test_junit_upload_http_error(respx_mock: respx.MockRouter) -> None: + respx_mock.post("/v1/repos/user/repo/ci_issues_upload").respond( + 422, + json={"detail": "CI Issues is not enabled on this repository"}, + ) + + with pytest.raises(httpx.HTTPStatusError): + await junit_upload_mod.upload( + "https://api.mergify.com/", + "token", + "user/repo", + "head-sha", + "ci-job", + "circleci", + (str(REPORT_XML),), + ) diff --git a/mergify_cli/utils.py b/mergify_cli/utils.py index 1a03b10..b85a09b 100644 --- a/mergify_cli/utils.py +++ b/mergify_cli/utils.py @@ -185,6 +185,37 @@ async def log_httpx_response(response: httpx.Response) -> None: ) +def get_http_client( + server: str, + headers: dict[str, typing.Any] | None = None, + event_hooks: typing.Mapping[str, list[typing.Callable[..., typing.Any]]] + | None = None, + follow_redirects: bool = False, +) -> httpx.AsyncClient: + default_headers = {"User-Agent": f"mergify_cli/{VERSION}"} + if headers is not None: + default_headers |= headers + + default_event_hooks: typing.Mapping[str, list[typing.Callable[..., typing.Any]]] = { + "request": [], + "response": [], + } + if event_hooks is not None: + default_event_hooks["request"] += event_hooks["request"] + default_event_hooks["response"] += event_hooks["response"] + if is_debug(): + default_event_hooks["request"].insert(0, log_httpx_request) + default_event_hooks["response"].insert(0, log_httpx_response) + + return httpx.AsyncClient( + base_url=server, + headers=default_headers, + event_hooks=default_event_hooks, + follow_redirects=follow_redirects, + timeout=5.0, + ) + + def get_github_http_client(github_server: str, token: str) -> httpx.AsyncClient: event_hooks: typing.Mapping[str, list[typing.Callable[..., typing.Any]]] = { "request": [], @@ -194,16 +225,14 @@ def get_github_http_client(github_server: str, token: str) -> httpx.AsyncClient: event_hooks["request"].insert(0, log_httpx_request) event_hooks["response"].insert(0, log_httpx_response) - return httpx.AsyncClient( - base_url=github_server, + return get_http_client( + github_server, headers={ "Accept": "application/vnd.github.v3+json", - "User-Agent": f"mergify_cli/{VERSION}", "Authorization": f"token {token}", }, event_hooks=event_hooks, follow_redirects=True, - timeout=5.0, )