Split compute and display gitlab ci config / diff ACIX-423 (#29686)

Co-authored-by: Nicolas Schweitzer <[email protected]>

.gitlab-ci.yml

@@ -1,7 +1,7 @@
 ---
 include:
   - .gitlab/.pre/cancel-prev-pipelines.yml
-  - .gitlab/.pre/test_gitlab_configuration.yml
+  - .gitlab/.pre/gitlab_configuration.yml
   - .gitlab/benchmarks/include.yml
   - .gitlab/binary_build/include.yml
   - .gitlab/check_deploy/check_deploy.yml

.gitlab/.pre/test_gitlab_configuration.yml → .gitlab/.pre/gitlab_configuration.yml

@@ -22,3 +22,28 @@ test_gitlab_compare_to:
     - !reference [.setup_agent_github_app]
     - pip install -r tasks/requirements.txt
     - inv pipeline.compare-to-itself
+
+# Computes and uploads the GitLab CI configuration diff as an artifact
+compute_gitlab_ci_config:
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
+  stage: .pre
+  needs: []
+  tags: ["arch:arm64"]
+  rules:
+    - if: $CI_PIPELINE_SOURCE != "push"
+      when: never
+    - when: on_success
+  before_script:
+    # Get main history
+    - git fetch origin main
+    - git checkout main
+    - git checkout -
+  script:
+    - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_FULL_API_TOKEN) || exit $?; export GITLAB_TOKEN
+    - mkdir -p artifacts
+    - inv -e gitlab.compute-gitlab-ci-config --before-file artifacts/before.gitlab-ci.yml --after-file artifacts/after.gitlab-ci.yml --diff-file artifacts/diff.gitlab-ci.yml
+  artifacts:
+    when: always
+    paths:
+      - artifacts/
+    expire_in: 1 day

.gitlab/notify/notify.yml

@@ -91,7 +91,7 @@ notify_github:
 notify_gitlab_ci_changes:
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   stage: notify
-  needs: []
+  needs: [compute_gitlab_ci_config]
   tags: ["arch:amd64"]
   rules:
     - if: $CI_PIPELINE_SOURCE != "push"
@@ -101,16 +101,11 @@ notify_gitlab_ci_changes:
           - .gitlab-ci.yml
           - .gitlab/**/*.yml
         compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
-  before_script:
-    # Get main history
-    - git fetch origin main
-    - git checkout main
-    - git checkout -
   script:
     - python3 -m pip install -r tasks/libs/requirements-github.txt
     - !reference [.setup_agent_github_app]
     - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_FULL_API_TOKEN) || exit $?; export GITLAB_TOKEN
-    - inv -e notify.gitlab-ci-diff --pr-comment
+    - inv -e notify.gitlab-ci-diff --from-diff artifacts/diff.gitlab-ci.yml --pr-comment
 
 .failure_summary_job:
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
@@ -163,14 +158,14 @@ notify_failure_summary_daily:
 
 close_failing_tests_stale_issues:
   stage: notify
-  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   rules:
     # Daily
     - if: $CI_COMMIT_BRANCH != "main" || $CI_PIPELINE_SOURCE != "schedule"
       when: never
     - !reference [.on_deploy_nightly_repo_branch_always]
   needs: []
-  tags: ["arch:amd64"]
+  tags: ["arch:arm64"]
   script:
     - weekday="$(date --utc '+%A')"
     # Weekly on Friday

tasks/gitlab_helpers.py

@@ -14,6 +14,7 @@
 
 from tasks.kernel_matrix_testing.ci import get_kmt_dashboard_links
 from tasks.libs.ciproviders.gitlab_api import (
+    compute_gitlab_ci_config_diff,
     get_all_gitlab_ci_configurations,
     get_gitlab_ci_configuration,
     get_gitlab_repo,
@@ -281,3 +282,31 @@ def print_entry_points(ctx):
     print(len(entry_points), 'entry points:')
     for entry_point, config in entry_points.items():
         print(f'- {color_message(entry_point, Color.BOLD)} ({len(config)} components)')
+
+
+@task
+def compute_gitlab_ci_config(
+    ctx,
+    before: str | None = None,
+    after: str | None = None,
+    before_file: str = 'before.gitlab-ci.yml',
+    after_file: str = 'after.gitlab-ci.yml',
+    diff_file: str = 'diff.gitlab-ci.yml',
+):
+    """
+    Will compute the Gitlab CI full configuration for the current commit and the base commit and will compute the diff between them.
+    """
+
+    before_config, after_config, diff = compute_gitlab_ci_config_diff(ctx, before, after)
+
+    print('Writing', before_file)
+    with open(before_file, 'w') as f:
+        f.write(yaml.safe_dump(before_config))
+
+    print('Writing', after_file)
+    with open(after_file, 'w') as f:
+        f.write(yaml.safe_dump(after_config))
+
+    print('Writing', diff_file)
+    with open(diff_file, 'w') as f:
+        f.write(yaml.safe_dump(diff.to_dict()))

tasks/libs/ciproviders/gitlab_api.py

@@ -18,6 +18,7 @@
 from invoke.exceptions import Exit
 
 from tasks.libs.common.color import Color, color_message
+from tasks.libs.common.constants import DEFAULT_BRANCH
 from tasks.libs.common.git import get_common_ancestor, get_current_branch
 from tasks.libs.common.utils import retry_function
 
@@ -116,20 +117,64 @@ def refresh_pipeline(pipeline: ProjectPipeline):
 
 
 class GitlabCIDiff:
-    def __init__(self, before: dict, after: dict) -> None:
+    def __init__(
+        self,
+        before: dict | None = None,
+        after: dict | None = None,
+        added: set[str] | None = None,
+        removed: set[str] | None = None,
+        modified: set[str] | None = None,
+        renamed: set[tuple[str, str]] | None = None,
+        modified_diffs: dict[str, list[str]] | None = None,
+        added_contents: dict[str, str] | None = None,
+    ) -> None:
         """
         Used to display job diffs between two gitlab ci configurations
         """
-        self.before = before
-        self.after = after
-        self.added_contents = {}
-        self.modified_diffs = {}
-
-        self.make_diff()
+        self.before = before or {}
+        self.after = after or {}
+        self.added = added or set()
+        self.removed = removed or set()
+        self.modified = modified or set()
+        self.renamed = renamed or set()
+        self.modified_diffs = modified_diffs or {}
+        self.added_contents = added_contents or {}
 
     def __bool__(self) -> bool:
         return bool(self.added or self.removed or self.modified or self.renamed)
 
+    def to_dict(self) -> dict:
+        return {
+            'before': self.before,
+            'after': self.after,
+            'added': self.added,
+            'removed': self.removed,
+            'modified': self.modified,
+            'renamed': list(self.renamed),
+            'modied_diffs': self.modified_diffs,
+            'added_contents': self.added_contents,
+        }
+
+    @staticmethod
+    def from_dict(data: dict) -> GitlabCIDiff:
+        return GitlabCIDiff(
+            before=data['before'],
+            after=data['after'],
+            added=set(data['added']),
+            removed=set(data['removed']),
+            modified=set(data['modified']),
+            renamed=set(data['renamed']),
+            modified_diffs=data['modied_diffs'],
+            added_contents=data['added_contents'],
+        )
+
+    @staticmethod
+    def from_contents(before: dict | None = None, after: dict | None = None) -> GitlabCIDiff:
+        diff = GitlabCIDiff(before, after)
+        diff.make_diff()
+
+        return diff
+
     def make_diff(self):
         """
         Compute the diff between the two gitlab ci configurations
@@ -336,6 +381,27 @@ def str_note() -> list[str]:
 
         return '\n'.join(res)
 
+    def iter_jobs(self, added=True, modified=True, removed=False):
+        """
+        Will iterate over all jobs in all files for the given states
+
+        Returns a tuple of (job_name, contents, state)
+
+        Note that the contents of the job is the contents after modification if modified or before removal if removed
+        """
+
+        if added:
+            for job in self.added:
+                yield job, self.after[job], 'added'
+
+        if modified:
+            for job in self.modified:
+                yield job, self.after[job], 'modified'
+
+        if removed:
+            for job in self.removed:
+                yield job, self.before[job], 'removed'
+
 
 class MultiGitlabCIDiff:
     @dataclass
@@ -346,25 +412,59 @@ class MultiDiff:
         is_added: bool
         is_removed: bool
 
-    def __init__(self, before: dict[str, dict], after: dict[str, dict]) -> None:
+        def to_dict(self) -> dict:
+            return {
+                'entry_point': self.entry_point,
+                'diff': self.diff.to_dict(),
+                'is_added': self.is_added,
+                'is_removed': self.is_removed,
+            }
+
+        @staticmethod
+        def from_dict(data: dict) -> MultiGitlabCIDiff.MultiDiff:
+            return MultiGitlabCIDiff.MultiDiff(
+                data['entry_point'], GitlabCIDiff.from_dict(data['diff']), data['is_added'], data['is_removed']
+            )
+
+    def __init__(
+        self,
+        before: dict[str, dict] | None = None,
+        after: dict[str, dict] | None = None,
+        diffs: list[MultiGitlabCIDiff.MultiDiff] | None = None,
+    ) -> None:
         """
         Used to display job diffs between two full gitlab ci configurations (multiple entry points)
 
         - before/after: Dict of [entry point] -> ([job name] -> job content)
         """
-        self.before = dict(before)
-        self.after = dict(after)
-
-        self.diffs: list[MultiGitlabCIDiff.MultiDiff] = []
-
-        self.make_diff()
+        self.before = before
+        self.after = after
+        self.diffs = diffs or []
 
     def __bool__(self) -> bool:
         return bool(self.diffs)
 
+    def to_dict(self) -> dict:
+        return {'before': self.before, 'after': self.after, 'diffs': [diff.to_dict() for diff in self.diffs]}
+
+    @staticmethod
+    def from_dict(data: dict) -> MultiGitlabCIDiff:
+        return MultiGitlabCIDiff(
+            data['before'], data['after'], [MultiGitlabCIDiff.MultiDiff.from_dict(d) for d in data['diffs']]
+        )
+
+    @staticmethod
+    def from_contents(before: dict[str, dict] | None = None, after: dict[str, dict] | None = None) -> MultiGitlabCIDiff:
+        diff = MultiGitlabCIDiff(before, after)
+        diff.make_diff()
+
+        return diff
+
     def make_diff(self):
+        self.diffs = []
+
         for entry_point in set(list(self.before) + list(self.after)):
-            diff = GitlabCIDiff(self.before.get(entry_point, {}), self.after.get(entry_point, {}))
+            diff = GitlabCIDiff.from_contents(self.before.get(entry_point, {}), self.after.get(entry_point, {}))
 
             # Diff for this entry point, add it to the list
             if diff:
@@ -428,6 +528,19 @@ def str_entry_end() -> list[str]:
 
         return '\n'.join(res)
 
+    def iter_jobs(self, added=True, modified=True, removed=False):
+        """
+        Will iterate over all jobs in all files for the given states
+
+        Returns a tuple of (entry_point, job_name, contents, state)
+
+        Note that the contents is the contents after modification or before removal
+        """
+
+        for diff in self.diffs:
+            for job, contents, state in diff.diff.iter_jobs(added=added, modified=modified, removed=removed):
+                yield diff.entry_point, job, contents, state
+
 
 class ReferenceTag(yaml.YAMLObject):
     """
@@ -1048,3 +1161,27 @@ def gitlab_configuration_is_modified(ctx):
             return True
 
     return False
+
+
+def compute_gitlab_ci_config_diff(ctx, before: str, after: str):
+    """
+    Computes the full configs and the diff between two git references.
+    The "after reference" is compared to the Lowest Common Ancestor (LCA) commit of "before reference" and "after reference".
+    """
+
+    before_name = before or "merge base"
+    after_name = after or "local files"
+
+    # The before commit is the LCA commit between before and after
+    before = before or DEFAULT_BRANCH
+    before = get_common_ancestor(ctx, before, after or "HEAD")
+
+    print(f'Getting after changes config ({color_message(after_name, Color.BOLD)})')
+    after_config = get_all_gitlab_ci_configurations(ctx, git_ref=after, clean_configs=True)
+
+    print(f'Getting before changes config ({color_message(before_name, Color.BOLD)})')
+    before_config = get_all_gitlab_ci_configurations(ctx, git_ref=before, clean_configs=True)
+
+    diff = MultiGitlabCIDiff.from_contents(before_config, after_config)
+
+    return before_config, after_config, diff

tasks/libs/common/git.py

@@ -54,8 +54,8 @@ def get_current_branch(ctx) -> str:
     return ctx.run("git rev-parse --abbrev-ref HEAD", hide=True).stdout.strip()
 
 
-def get_common_ancestor(ctx, branch) -> str:
-    return ctx.run(f"git merge-base {branch} main", hide=True).stdout.strip()
+def get_common_ancestor(ctx, branch, base=DEFAULT_BRANCH) -> str:
+    return ctx.run(f"git merge-base {branch} {base}", hide=True).stdout.strip()
 
 
 def check_uncommitted_changes(ctx):