Skip to content

Commit

Permalink
infra: move generate_workflows_lib from contrib to core (open-telemet…
Browse files Browse the repository at this point in the history
  • Loading branch information
emdneto authored Dec 3, 2024
1 parent 715ec63 commit 415c94f
Show file tree
Hide file tree
Showing 6 changed files with 351 additions and 17 deletions.
214 changes: 200 additions & 14 deletions .github/workflows/generate_workflows.py
Original file line number Diff line number Diff line change
@@ -1,19 +1,205 @@
from collections import defaultdict
from pathlib import Path
from re import compile as re_compile

from generate_workflows_lib import (
generate_lint_workflow,
generate_misc_workflow,
generate_test_workflow,
from jinja2 import Environment, FileSystemLoader
from tox.config.cli.parse import get_options
from tox.config.sets import CoreConfigSet
from tox.config.source.tox_ini import ToxIni
from tox.session.state import State

_tox_test_env_regex = re_compile(
r"(?P<python_version>py\w+)-test-"
r"(?P<name>[-\w]+\w)-?(?P<test_requirements>\d+)?"
)
_tox_lint_env_regex = re_compile(r"lint-(?P<name>[-\w]+)")
_tox_contrib_env_regex = re_compile(
r"py38-test-(?P<name>[-\w]+\w)-?(?P<contrib_requirements>\d+)?"
)

tox_ini_path = Path(__file__).parent.parent.parent.joinpath("tox.ini")
workflows_directory_path = Path(__file__).parent

generate_test_workflow(
tox_ini_path,
workflows_directory_path,
"ubuntu-latest",
"windows-latest",
)
generate_lint_workflow(tox_ini_path, workflows_directory_path)
generate_misc_workflow(tox_ini_path, workflows_directory_path)
def get_tox_envs(tox_ini_path: Path) -> list:
tox_ini = ToxIni(tox_ini_path)

conf = State(get_options(), []).conf

tox_section = next(tox_ini.sections())

core_config_set = CoreConfigSet(
conf, tox_section, tox_ini_path.parent, tox_ini_path
)

(
core_config_set.loaders.extend(
tox_ini.get_loaders(
tox_section,
base=[],
override_map=defaultdict(list, {}),
conf=core_config_set,
)
)
)

return core_config_set.load("env_list")


def get_test_job_datas(tox_envs: list, operating_systems: list) -> list:
os_alias = {"ubuntu-latest": "Ubuntu", "windows-latest": "Windows"}

python_version_alias = {
"pypy3": "pypy-3.8",
"py38": "3.8",
"py39": "3.9",
"py310": "3.10",
"py311": "3.11",
"py312": "3.12",
}

test_job_datas = []

for operating_system in operating_systems:
for tox_env in tox_envs:
tox_test_env_match = _tox_test_env_regex.match(tox_env)

if tox_test_env_match is None:
continue

groups = tox_test_env_match.groupdict()

aliased_python_version = python_version_alias[
groups["python_version"]
]
tox_env = tox_test_env_match.string

test_requirements = groups["test_requirements"]

if test_requirements is None:
test_requirements = " "

else:
test_requirements = f"-{test_requirements} "

test_job_datas.append(
{
"name": f"{tox_env}_{operating_system}",
"ui_name": (
f"{groups['name']}"
f"{test_requirements}"
f"{aliased_python_version} "
f"{os_alias[operating_system]}"
),
"python_version": aliased_python_version,
"tox_env": tox_env,
"os": operating_system,
}
)

return test_job_datas


def get_lint_job_datas(tox_envs: list) -> list:
lint_job_datas = []

for tox_env in tox_envs:
tox_lint_env_match = _tox_lint_env_regex.match(tox_env)

if tox_lint_env_match is None:
continue

tox_env = tox_lint_env_match.string

lint_job_datas.append(
{
"name": f"{tox_env}",
"ui_name": f"{tox_lint_env_match.groupdict()['name']}",
"tox_env": tox_env,
}
)

return lint_job_datas


def get_misc_job_datas(tox_envs: list) -> list:
regex_patterns = [
_tox_test_env_regex,
_tox_lint_env_regex,
_tox_contrib_env_regex,
re_compile(r"benchmark.+"),
]

return [
tox_env
for tox_env in tox_envs
if not any(pattern.match(tox_env) for pattern in regex_patterns)
]


def _generate_workflow(
job_datas: list,
template_name: str,
output_dir: Path,
max_jobs: int = 250,
):
# Github seems to limit the amount of jobs in a workflow file, that is why
# they are split in groups of 250 per workflow file.
for file_number, job_datas in enumerate(
[
job_datas[index : index + max_jobs]
for index in range(0, len(job_datas), max_jobs)
]
):
with open(
output_dir.joinpath(f"{template_name}_{file_number}.yml"), "w"
) as test_yml_file:
test_yml_file.write(
Environment(
loader=FileSystemLoader(
Path(__file__).parent.joinpath("templates")
)
)
.get_template(f"{template_name}.yml.j2")
.render(job_datas=job_datas, file_number=file_number)
)
test_yml_file.write("\n")


def generate_test_workflow(
tox_ini_path: Path, workflow_directory_path: Path, operating_systems
) -> None:
_generate_workflow(
get_test_job_datas(get_tox_envs(tox_ini_path), operating_systems),
"test",
workflow_directory_path,
)


def generate_lint_workflow(
tox_ini_path: Path,
workflow_directory_path: Path,
) -> None:
_generate_workflow(
get_lint_job_datas(get_tox_envs(tox_ini_path)),
"lint",
workflow_directory_path,
)


def generate_misc_workflow(
tox_ini_path: Path,
workflow_directory_path: Path,
) -> None:
_generate_workflow(
get_misc_job_datas(get_tox_envs(tox_ini_path)),
"misc",
workflow_directory_path,
)


if __name__ == "__main__":
tox_ini_path = Path(__file__).parent.parent.parent.joinpath("tox.ini")
output_dir = Path(__file__).parent
generate_test_workflow(
tox_ini_path, output_dir, ["ubuntu-latest", "windows-latest"]
)
generate_lint_workflow(tox_ini_path, output_dir)
generate_misc_workflow(tox_ini_path, output_dir)
2 changes: 1 addition & 1 deletion .github/workflows/misc_0.yml
Original file line number Diff line number Diff line change
Expand Up @@ -233,7 +233,7 @@ jobs:
- name: Run tests
run: tox -e generate-workflows

- name: Check workflows are up to date
- name: Check github workflows are up to date
run: git diff --exit-code || (echo 'Generated workflows are out of date, run "tox -e generate-workflows" and commit the changes in this PR.' && exit 1)

ruff:
Expand Down
37 changes: 37 additions & 0 deletions .github/workflows/templates/lint.yml.j2
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
# Do not edit this file.
# This file is generated automatically by executing tox -e generate-workflows

name: Lint {{ file_number }}

on:
push:
branches-ignore:
- 'release/*'
pull_request:

env:
CORE_REPO_SHA: main
CONTRIB_REPO_SHA: main
PIP_EXISTS_ACTION: w

jobs:
{%- for job_data in job_datas %}

{{ job_data.name }}:
name: {{ job_data.ui_name }}
runs-on: ubuntu-latest
steps:
- name: Checkout repo @ SHA - ${% raw %}{{ github.sha }}{% endraw %}
uses: actions/checkout@v4

- name: Set up Python 3.12
uses: actions/setup-python@v5
with:
python-version: "3.12"

- name: Install tox
run: pip install tox

- name: Run tests
run: tox -e {{ job_data.tox_env }}
{%- endfor %}
69 changes: 69 additions & 0 deletions .github/workflows/templates/misc.yml.j2
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
# Do not edit this file.
# This file is generated automatically by executing tox -e generate-workflows

name: Misc {{ file_number }}

on:
push:
branches-ignore:
- 'release/*'
pull_request:

env:
CORE_REPO_SHA: main
CONTRIB_REPO_SHA: main
PIP_EXISTS_ACTION: w

jobs:
{%- for job_data in job_datas %}

{{ job_data }}:
name: {{ job_data }}
runs-on: ubuntu-latest
{%- if job_data == "generate-workflows" %}
if: |
!contains(github.event.pull_request.labels.*.name, 'Skip generate-workflows')
&& github.event.pull_request.user.login != 'opentelemetrybot' && github.event_name == 'pull_request'
{%- endif %}
{%- if job_data == "public-symbols-check" %}
if: |
!contains(github.event.pull_request.labels.*.name, 'Approve Public API check')
&& github.actor != 'opentelemetrybot' && github.event_name == 'pull_request'
{%- endif %}
{%- if job_data == "docs" %}
if: |
github.event.pull_request.user.login != 'opentelemetrybot' && github.event_name == 'pull_request'
{%- endif %}
steps:
- name: Checkout repo @ SHA - ${% raw %}{{ github.sha }}{% endraw %}
uses: actions/checkout@v4
{%- if job_data == "public-symbols-check" %}
with:
fetch-depth: 0

- name: Checkout main
run: git checkout main

- name: Pull origin
run: git pull --rebase=false origin main

- name: Checkout pull request
run: git checkout ${% raw %}{{ github.event.pull_request.head.sha }}{% endraw %}
{%- endif %}

- name: Set up Python 3.11
uses: actions/setup-python@v5
with:
python-version: "3.11"

- name: Install tox
run: pip install tox

- name: Run tests
run: tox -e {{ job_data }}
{%- if job_data == "generate-workflows" %}

- name: Check github workflows are up to date
run: git diff --exit-code || (echo 'Generated workflows are out of date, run "tox -e generate-workflows" and commit the changes in this PR.' && exit 1)
{%- endif %}
{%- endfor %}
42 changes: 42 additions & 0 deletions .github/workflows/templates/test.yml.j2
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
# Do not edit this file.
# This file is generated automatically by executing tox -e generate-workflows

name: Test {{ file_number }}

on:
push:
branches-ignore:
- 'release/*'
pull_request:

env:
CORE_REPO_SHA: main
CONTRIB_REPO_SHA: main
PIP_EXISTS_ACTION: w

jobs:
{%- for job_data in job_datas %}

{{ job_data.name }}:
name: {{ job_data.ui_name }}
runs-on: {{ job_data.os }}
steps:
- name: Checkout repo @ SHA - ${% raw %}{{ github.sha }}{% endraw %}
uses: actions/checkout@v4

- name: Set up Python {{ job_data.python_version }}
uses: actions/setup-python@v5
with:
python-version: "{{ job_data.python_version }}"

- name: Install tox
run: pip install tox
{%- if job_data.os == "windows-latest" %}

- name: Configure git to support long filenames
run: git config --system core.longpaths true
{%- endif %}

- name: Run tests
run: tox -e {{ job_data.tox_env }} -- -ra
{%- endfor %}
4 changes: 2 additions & 2 deletions tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -334,9 +334,9 @@ commands =
python {toxinidir}/scripts/public_symbols_checker.py

[testenv:generate-workflows]
recreate = True
deps =
{env:CONTRIB_REPO}\#egg=generate_workflows_lib&subdirectory=.github/workflows/generate_workflows_lib
tox
Jinja2
commands =
python {toxinidir}/.github/workflows/generate_workflows.py

Expand Down

0 comments on commit 415c94f

Please sign in to comment.