diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 75e0d5bb34bc6..a98e1ba9c7e81 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,7 @@ --- include: - .gitlab/.pre/cancel-prev-pipelines.yml - - .gitlab/.pre/test_gitlab_configuration.yml + - .gitlab/.pre/gitlab_configuration.yml - .gitlab/benchmarks/include.yml - .gitlab/binary_build/include.yml - .gitlab/check_deploy/check_deploy.yml diff --git a/.gitlab/.pre/test_gitlab_configuration.yml b/.gitlab/.pre/gitlab_configuration.yml similarity index 53% rename from .gitlab/.pre/test_gitlab_configuration.yml rename to .gitlab/.pre/gitlab_configuration.yml index 1de3d176087d5..2114c9aaffb95 100644 --- a/.gitlab/.pre/test_gitlab_configuration.yml +++ b/.gitlab/.pre/gitlab_configuration.yml @@ -22,3 +22,28 @@ test_gitlab_compare_to: - !reference [.setup_agent_github_app] - pip install -r tasks/requirements.txt - inv pipeline.compare-to-itself + +# Computes and uploads the GitLab CI configuration diff as an artifact +compute_gitlab_ci_config: + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + stage: .pre + needs: [] + tags: ["arch:arm64"] + rules: + - if: $CI_PIPELINE_SOURCE != "push" + when: never + - when: on_success + before_script: + # Get main history + - git fetch origin main + - git checkout main + - git checkout - + script: + - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_FULL_API_TOKEN) || exit $?; export GITLAB_TOKEN + - mkdir -p artifacts + - inv -e gitlab.compute-gitlab-ci-config --before-file artifacts/before.gitlab-ci.yml --after-file artifacts/after.gitlab-ci.yml --diff-file artifacts/diff.gitlab-ci.yml + artifacts: + when: always + paths: + - artifacts/ + expire_in: 1 day diff --git a/.gitlab/notify/notify.yml b/.gitlab/notify/notify.yml index 83015533b1dfc..ea966d752ef63 100644 --- a/.gitlab/notify/notify.yml +++ b/.gitlab/notify/notify.yml @@ -91,7 +91,7 @@ notify_github: notify_gitlab_ci_changes: image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES stage: notify - needs: [] + needs: [compute_gitlab_ci_config] tags: ["arch:amd64"] rules: - if: $CI_PIPELINE_SOURCE != "push" @@ -101,16 +101,11 @@ notify_gitlab_ci_changes: - .gitlab-ci.yml - .gitlab/**/*.yml compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916 - before_script: - # Get main history - - git fetch origin main - - git checkout main - - git checkout - script: - python3 -m pip install -r tasks/libs/requirements-github.txt - !reference [.setup_agent_github_app] - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_FULL_API_TOKEN) || exit $?; export GITLAB_TOKEN - - inv -e notify.gitlab-ci-diff --pr-comment + - inv -e notify.gitlab-ci-diff --from-diff artifacts/diff.gitlab-ci.yml --pr-comment .failure_summary_job: image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES @@ -163,14 +158,14 @@ notify_failure_summary_daily: close_failing_tests_stale_issues: stage: notify - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES rules: # Daily - if: $CI_COMMIT_BRANCH != "main" || $CI_PIPELINE_SOURCE != "schedule" when: never - !reference [.on_deploy_nightly_repo_branch_always] needs: [] - tags: ["arch:amd64"] + tags: ["arch:arm64"] script: - weekday="$(date --utc '+%A')" # Weekly on Friday diff --git a/tasks/gitlab_helpers.py b/tasks/gitlab_helpers.py index c65107292fb79..26aa05b84b157 100644 --- a/tasks/gitlab_helpers.py +++ b/tasks/gitlab_helpers.py @@ -14,6 +14,7 @@ from tasks.kernel_matrix_testing.ci import get_kmt_dashboard_links from tasks.libs.ciproviders.gitlab_api import ( + compute_gitlab_ci_config_diff, get_all_gitlab_ci_configurations, get_gitlab_ci_configuration, get_gitlab_repo, @@ -281,3 +282,31 @@ def print_entry_points(ctx): print(len(entry_points), 'entry points:') for entry_point, config in entry_points.items(): print(f'- {color_message(entry_point, Color.BOLD)} ({len(config)} components)') + + +@task +def compute_gitlab_ci_config( + ctx, + before: str | None = None, + after: str | None = None, + before_file: str = 'before.gitlab-ci.yml', + after_file: str = 'after.gitlab-ci.yml', + diff_file: str = 'diff.gitlab-ci.yml', +): + """ + Will compute the Gitlab CI full configuration for the current commit and the base commit and will compute the diff between them. + """ + + before_config, after_config, diff = compute_gitlab_ci_config_diff(ctx, before, after) + + print('Writing', before_file) + with open(before_file, 'w') as f: + f.write(yaml.safe_dump(before_config)) + + print('Writing', after_file) + with open(after_file, 'w') as f: + f.write(yaml.safe_dump(after_config)) + + print('Writing', diff_file) + with open(diff_file, 'w') as f: + f.write(yaml.safe_dump(diff.to_dict())) diff --git a/tasks/libs/ciproviders/gitlab_api.py b/tasks/libs/ciproviders/gitlab_api.py index c11e16c1275bc..3a416b92574f4 100644 --- a/tasks/libs/ciproviders/gitlab_api.py +++ b/tasks/libs/ciproviders/gitlab_api.py @@ -18,6 +18,7 @@ from invoke.exceptions import Exit from tasks.libs.common.color import Color, color_message +from tasks.libs.common.constants import DEFAULT_BRANCH from tasks.libs.common.git import get_common_ancestor, get_current_branch from tasks.libs.common.utils import retry_function @@ -116,20 +117,64 @@ def refresh_pipeline(pipeline: ProjectPipeline): class GitlabCIDiff: - def __init__(self, before: dict, after: dict) -> None: + def __init__( + self, + before: dict | None = None, + after: dict | None = None, + added: set[str] | None = None, + removed: set[str] | None = None, + modified: set[str] | None = None, + renamed: set[tuple[str, str]] | None = None, + modified_diffs: dict[str, list[str]] | None = None, + added_contents: dict[str, str] | None = None, + ) -> None: """ Used to display job diffs between two gitlab ci configurations """ - self.before = before - self.after = after - self.added_contents = {} - self.modified_diffs = {} - - self.make_diff() + self.before = before or {} + self.after = after or {} + self.added = added or set() + self.removed = removed or set() + self.modified = modified or set() + self.renamed = renamed or set() + self.modified_diffs = modified_diffs or {} + self.added_contents = added_contents or {} def __bool__(self) -> bool: return bool(self.added or self.removed or self.modified or self.renamed) + def to_dict(self) -> dict: + return { + 'before': self.before, + 'after': self.after, + 'added': self.added, + 'removed': self.removed, + 'modified': self.modified, + 'renamed': list(self.renamed), + 'modied_diffs': self.modified_diffs, + 'added_contents': self.added_contents, + } + + @staticmethod + def from_dict(data: dict) -> GitlabCIDiff: + return GitlabCIDiff( + before=data['before'], + after=data['after'], + added=set(data['added']), + removed=set(data['removed']), + modified=set(data['modified']), + renamed=set(data['renamed']), + modified_diffs=data['modied_diffs'], + added_contents=data['added_contents'], + ) + + @staticmethod + def from_contents(before: dict | None = None, after: dict | None = None) -> GitlabCIDiff: + diff = GitlabCIDiff(before, after) + diff.make_diff() + + return diff + def make_diff(self): """ Compute the diff between the two gitlab ci configurations @@ -336,6 +381,27 @@ def str_note() -> list[str]: return '\n'.join(res) + def iter_jobs(self, added=True, modified=True, removed=False): + """ + Will iterate over all jobs in all files for the given states + + Returns a tuple of (job_name, contents, state) + + Note that the contents of the job is the contents after modification if modified or before removal if removed + """ + + if added: + for job in self.added: + yield job, self.after[job], 'added' + + if modified: + for job in self.modified: + yield job, self.after[job], 'modified' + + if removed: + for job in self.removed: + yield job, self.before[job], 'removed' + class MultiGitlabCIDiff: @dataclass @@ -346,25 +412,59 @@ class MultiDiff: is_added: bool is_removed: bool - def __init__(self, before: dict[str, dict], after: dict[str, dict]) -> None: + def to_dict(self) -> dict: + return { + 'entry_point': self.entry_point, + 'diff': self.diff.to_dict(), + 'is_added': self.is_added, + 'is_removed': self.is_removed, + } + + @staticmethod + def from_dict(data: dict) -> MultiGitlabCIDiff.MultiDiff: + return MultiGitlabCIDiff.MultiDiff( + data['entry_point'], GitlabCIDiff.from_dict(data['diff']), data['is_added'], data['is_removed'] + ) + + def __init__( + self, + before: dict[str, dict] | None = None, + after: dict[str, dict] | None = None, + diffs: list[MultiGitlabCIDiff.MultiDiff] | None = None, + ) -> None: """ Used to display job diffs between two full gitlab ci configurations (multiple entry points) - before/after: Dict of [entry point] -> ([job name] -> job content) """ - self.before = dict(before) - self.after = dict(after) - - self.diffs: list[MultiGitlabCIDiff.MultiDiff] = [] - - self.make_diff() + self.before = before + self.after = after + self.diffs = diffs or [] def __bool__(self) -> bool: return bool(self.diffs) + def to_dict(self) -> dict: + return {'before': self.before, 'after': self.after, 'diffs': [diff.to_dict() for diff in self.diffs]} + + @staticmethod + def from_dict(data: dict) -> MultiGitlabCIDiff: + return MultiGitlabCIDiff( + data['before'], data['after'], [MultiGitlabCIDiff.MultiDiff.from_dict(d) for d in data['diffs']] + ) + + @staticmethod + def from_contents(before: dict[str, dict] | None = None, after: dict[str, dict] | None = None) -> MultiGitlabCIDiff: + diff = MultiGitlabCIDiff(before, after) + diff.make_diff() + + return diff + def make_diff(self): + self.diffs = [] + for entry_point in set(list(self.before) + list(self.after)): - diff = GitlabCIDiff(self.before.get(entry_point, {}), self.after.get(entry_point, {})) + diff = GitlabCIDiff.from_contents(self.before.get(entry_point, {}), self.after.get(entry_point, {})) # Diff for this entry point, add it to the list if diff: @@ -428,6 +528,19 @@ def str_entry_end() -> list[str]: return '\n'.join(res) + def iter_jobs(self, added=True, modified=True, removed=False): + """ + Will iterate over all jobs in all files for the given states + + Returns a tuple of (entry_point, job_name, contents, state) + + Note that the contents is the contents after modification or before removal + """ + + for diff in self.diffs: + for job, contents, state in diff.diff.iter_jobs(added=added, modified=modified, removed=removed): + yield diff.entry_point, job, contents, state + class ReferenceTag(yaml.YAMLObject): """ @@ -1048,3 +1161,27 @@ def gitlab_configuration_is_modified(ctx): return True return False + + +def compute_gitlab_ci_config_diff(ctx, before: str, after: str): + """ + Computes the full configs and the diff between two git references. + The "after reference" is compared to the Lowest Common Ancestor (LCA) commit of "before reference" and "after reference". + """ + + before_name = before or "merge base" + after_name = after or "local files" + + # The before commit is the LCA commit between before and after + before = before or DEFAULT_BRANCH + before = get_common_ancestor(ctx, before, after or "HEAD") + + print(f'Getting after changes config ({color_message(after_name, Color.BOLD)})') + after_config = get_all_gitlab_ci_configurations(ctx, git_ref=after, clean_configs=True) + + print(f'Getting before changes config ({color_message(before_name, Color.BOLD)})') + before_config = get_all_gitlab_ci_configurations(ctx, git_ref=before, clean_configs=True) + + diff = MultiGitlabCIDiff.from_contents(before_config, after_config) + + return before_config, after_config, diff diff --git a/tasks/libs/common/git.py b/tasks/libs/common/git.py index c3c6781a98bb8..8363f08ad9496 100644 --- a/tasks/libs/common/git.py +++ b/tasks/libs/common/git.py @@ -54,8 +54,8 @@ def get_current_branch(ctx) -> str: return ctx.run("git rev-parse --abbrev-ref HEAD", hide=True).stdout.strip() -def get_common_ancestor(ctx, branch) -> str: - return ctx.run(f"git merge-base {branch} main", hide=True).stdout.strip() +def get_common_ancestor(ctx, branch, base=DEFAULT_BRANCH) -> str: + return ctx.run(f"git merge-base {branch} {base}", hide=True).stdout.strip() def check_uncommitted_changes(ctx): diff --git a/tasks/notify.py b/tasks/notify.py index ec199f66f01cc..7cdd97f12b170 100644 --- a/tasks/notify.py +++ b/tasks/notify.py @@ -5,17 +5,17 @@ import sys from datetime import timedelta +import yaml from invoke import Context, task from invoke.exceptions import Exit import tasks.libs.notify.unit_tests as unit_tests_utils from tasks.github_tasks import pr_commenter +from tasks.gitlab_helpers import compute_gitlab_ci_config_diff from tasks.libs.ciproviders.gitlab_api import ( MultiGitlabCIDiff, - get_all_gitlab_ci_configurations, ) from tasks.libs.common.color import Color, color_message -from tasks.libs.common.constants import DEFAULT_BRANCH from tasks.libs.common.datadog_api import send_metrics from tasks.libs.common.utils import gitlab_section from tasks.libs.notify import alerts, failure_summary, pipeline_status @@ -138,7 +138,9 @@ def unit_tests(ctx, pipeline_id, pipeline_url, branch_name, dry_run=False): @task -def gitlab_ci_diff(ctx, before: str | None = None, after: str | None = None, pr_comment: bool = False): +def gitlab_ci_diff( + ctx, before: str | None = None, after: str | None = None, pr_comment: bool = False, from_diff: str | None = None +): """ Creates a diff from two gitlab-ci configurations. @@ -168,20 +170,12 @@ def gitlab_ci_diff(ctx, before: str | None = None, after: str | None = None, pr_ job_url = os.environ['CI_JOB_URL'] try: - before_name = before or "merge base" - after_name = after or "local files" - - # The before commit is the LCA commit between before and after - before = before or DEFAULT_BRANCH - before = ctx.run(f'git merge-base {before} {after or "HEAD"}', hide=True).stdout.strip() - - print(f'Getting after changes config ({color_message(after_name, Color.BOLD)})') - after_config = get_all_gitlab_ci_configurations(ctx, git_ref=after, clean_configs=True) - - print(f'Getting before changes config ({color_message(before_name, Color.BOLD)})') - before_config = get_all_gitlab_ci_configurations(ctx, git_ref=before, clean_configs=True) - - diff = MultiGitlabCIDiff(before_config, after_config) + if from_diff: + with open(from_diff) as f: + diff_data = yaml.safe_load(f) + diff = MultiGitlabCIDiff.from_dict(diff_data) + else: + _, _, diff = compute_gitlab_ci_config_diff(ctx, before, after) if not diff: print(color_message("No changes in the gitlab-ci configuration", Color.GREEN)) diff --git a/tasks/unit_tests/gitlab_api_tests.py b/tasks/unit_tests/gitlab_api_tests.py index 28f35457379dd..5f2add57d44f8 100644 --- a/tasks/unit_tests/gitlab_api_tests.py +++ b/tasks/unit_tests/gitlab_api_tests.py @@ -6,6 +6,7 @@ from tasks.libs.ciproviders.gitlab_api import ( GitlabCIDiff, + MultiGitlabCIDiff, clean_gitlab_ci_configuration, expand_matrix_jobs, filter_gitlab_ci_configuration, @@ -170,13 +171,56 @@ def test_make_diff(self): 'script': 'echo "???"', }, } - diff = GitlabCIDiff(before, after) + diff = GitlabCIDiff.from_contents(before, after) self.assertSetEqual(diff.modified, {'job1'}) self.assertSetEqual(set(diff.modified_diffs.keys()), {'job1'}) self.assertSetEqual(diff.removed, {'job4'}) self.assertSetEqual(diff.added, {'job5'}) self.assertSetEqual(diff.renamed, {('job2', 'job2_renamed')}) + def test_serialization(self): + before = { + 'job1': { + 'script': [ + 'echo "hello"', + 'echo "hello?"', + 'echo "hello!"', + ] + }, + 'job2': { + 'script': 'echo "world"', + }, + 'job3': { + 'script': 'echo "!"', + }, + 'job4': { + 'script': 'echo "?"', + }, + } + after = { + 'job1': { + 'script': [ + 'echo "hello"', + 'echo "bonjour?"', + 'echo "hello!"', + ] + }, + 'job2_renamed': { + 'script': 'echo "world"', + }, + 'job3': { + 'script': 'echo "!"', + }, + 'job5': { + 'script': 'echo "???"', + }, + } + diff = MultiGitlabCIDiff.from_contents({'file': before}, {'file': after}) + dict_diff = diff.to_dict() + diff_from_dict = MultiGitlabCIDiff.from_dict(dict_diff) + + self.assertDictEqual(diff_from_dict.before, diff.before) + class TestRetrieveAllPaths(unittest.TestCase): def test_all_configs(self):