diff --git a/tasks/__init__.py b/tasks/__init__.py index 12753e9a67e68..93ce84f1c37b2 100644 --- a/tasks/__init__.py +++ b/tasks/__init__.py @@ -62,6 +62,7 @@ trace_agent, vim, vscode, + worktree, ) from tasks.build_tags import audit_tag_impact, print_default_build_tags from tasks.components import lint_components, lint_fxutil_oneshot_test @@ -210,6 +211,7 @@ ns.add_collection(collector) ns.add_collection(invoke_unit_tests) ns.add_collection(debug) +ns.add_collection(worktree) ns.configure( { "run": { diff --git a/tasks/git.py b/tasks/git.py index 658c6615cd621..8125035b04320 100644 --- a/tasks/git.py +++ b/tasks/git.py @@ -4,15 +4,19 @@ from invoke.exceptions import Exit from tasks.libs.common.color import color_message -from tasks.libs.common.git import get_current_branch +from tasks.libs.common.git import get_current_branch, get_default_branch @task def check_protected_branch(ctx): local_branch = get_current_branch(ctx) - if local_branch == 'main': - print(color_message("You're about to commit or push to the main, are you sure this is what you want?", "red")) + if local_branch == get_default_branch(): + print( + color_message( + f"You're about to commit or push to {get_default_branch()}, are you sure this is what you want?", "red" + ) + ) raise Exit(code=1) if re.fullmatch(r'^[0-9]+\.[0-9]+\.x$', local_branch): diff --git a/tasks/github_tasks.py b/tasks/github_tasks.py index b5723f232cd93..abe46fd6ff366 100644 --- a/tasks/github_tasks.py +++ b/tasks/github_tasks.py @@ -19,8 +19,9 @@ trigger_macos_workflow, ) from tasks.libs.common.color import color_message -from tasks.libs.common.constants import DEFAULT_BRANCH, DEFAULT_INTEGRATIONS_CORE_BRANCH +from tasks.libs.common.constants import DEFAULT_INTEGRATIONS_CORE_BRANCH from tasks.libs.common.datadog_api import create_gauge, send_event, send_metrics +from tasks.libs.common.git import get_default_branch from tasks.libs.common.junit_upload_core import repack_macos_junit_tar from tasks.libs.common.utils import get_git_pretty_ref from tasks.libs.owners.linter import codeowner_has_orphans, directory_has_packages_without_owner @@ -36,7 +37,7 @@ def concurrency_key(): current_ref = get_git_pretty_ref() # We want workflows to run to completion on the default branch and release branches - if re.search(rf'^({DEFAULT_BRANCH}|\d+\.\d+\.x)$', current_ref): + if re.search(rf'^({get_default_branch()}|\d+\.\d+\.x)$', current_ref): return None return current_ref @@ -68,7 +69,7 @@ def _trigger_macos_workflow(release, destination=None, retry_download=0, retry_i def trigger_macos( _, workflow_type="build", - datadog_agent_ref=DEFAULT_BRANCH, + datadog_agent_ref=None, release_version="nightly-a7", major_version="7", destination=".", @@ -79,6 +80,13 @@ def trigger_macos( test_washer=False, integrations_core_ref=DEFAULT_INTEGRATIONS_CORE_BRANCH, ): + """ + Args: + datadog_agent_ref: If None, will be the default branch. + """ + + datadog_agent_ref = datadog_agent_ref or get_default_branch() + if workflow_type == "build": conclusion = _trigger_macos_workflow( # Provide the release version to be able to fetch the associated diff --git a/tasks/gotest.py b/tasks/gotest.py index ceaee2f020bc2..3ef5ebc9de905 100644 --- a/tasks/gotest.py +++ b/tasks/gotest.py @@ -514,12 +514,13 @@ def get_modified_packages(ctx, build_tags=None, lint=False) -> list[GoModule]: modules_to_test[best_module_path] = GoModule(best_module_path, test_targets=[relative_target]) # Clean up duplicated paths to reduce Go test cmd length + default_modules = get_default_modules() for module in modules_to_test: modules_to_test[module].test_targets = clean_nested_paths(modules_to_test[module].test_targets) if ( len(modules_to_test[module].test_targets) >= WINDOWS_MAX_PACKAGES_NUMBER ): # With more packages we can reach the limit of the command line length on Windows - modules_to_test[module].test_targets = get_default_modules()[module].test_targets + modules_to_test[module].test_targets = default_modules[module].test_targets print("Running tests for the following modules:") for module in modules_to_test: @@ -752,16 +753,17 @@ def format_packages(ctx: Context, impacted_packages: set[str], build_tags: list[ packages = [f'{package.replace("github.com/DataDog/datadog-agent/", "./")}' for package in impacted_packages] modules_to_test = {} + default_modules = get_default_modules() for package in packages: module_path = get_go_module(package) # Check if the module is in the target list of the modules we want to test - if module_path not in get_default_modules() or not get_default_modules()[module_path].should_test(): + if module_path not in default_modules or not default_modules[module_path].should_test(): continue # Check if the package is in the target list of the module we want to test targeted = False - for target in get_default_modules()[module_path].test_targets: + for target in default_modules[module_path].test_targets: if normpath(os.path.join(module_path, target)) in package: targeted = True break @@ -784,12 +786,13 @@ def format_packages(ctx: Context, impacted_packages: set[str], build_tags: list[ modules_to_test[module_path] = GoModule(module_path, test_targets=[relative_target]) # Clean up duplicated paths to reduce Go test cmd length + default_modules = get_default_modules() for module in modules_to_test: modules_to_test[module].test_targets = clean_nested_paths(modules_to_test[module].test_targets) if ( len(modules_to_test[module].test_targets) >= WINDOWS_MAX_PACKAGES_NUMBER ): # With more packages we can reach the limit of the command line length on Windows - modules_to_test[module].test_targets = get_default_modules()[module].test_targets + modules_to_test[module].test_targets = default_modules[module].test_targets module_to_remove = [] # Clean up to avoid running tests on package with no Go files matching build tags diff --git a/tasks/libs/ciproviders/github_actions_tools.py b/tasks/libs/ciproviders/github_actions_tools.py index 8bef945f46460..9399302271b5d 100644 --- a/tasks/libs/ciproviders/github_actions_tools.py +++ b/tasks/libs/ciproviders/github_actions_tools.py @@ -11,13 +11,13 @@ from tasks.libs.ciproviders.github_api import GithubAPI from tasks.libs.common.color import color_message -from tasks.libs.common.utils import DEFAULT_BRANCH +from tasks.libs.common.git import get_default_branch def trigger_macos_workflow( workflow_name="macos.yaml", github_action_ref="master", - datadog_agent_ref=DEFAULT_BRANCH, + datadog_agent_ref=None, release_version=None, major_version=None, gitlab_pipeline_id=None, @@ -31,6 +31,8 @@ def trigger_macos_workflow( """ Trigger a workflow to build a MacOS Agent. """ + + datadog_agent_ref = datadog_agent_ref or get_default_branch() inputs = {} if datadog_agent_ref is not None: diff --git a/tasks/libs/ciproviders/github_api.py b/tasks/libs/ciproviders/github_api.py index 2d61567f5016b..f8ed33e3a3252 100644 --- a/tasks/libs/ciproviders/github_api.py +++ b/tasks/libs/ciproviders/github_api.py @@ -13,6 +13,7 @@ from tasks.libs.common.color import color_message from tasks.libs.common.constants import GITHUB_REPO_NAME +from tasks.libs.common.git import get_default_branch try: import semver @@ -494,7 +495,7 @@ def create_release_pr(title, base_branch, target_branch, version, changelog_pr=F ] if changelog_pr: - labels.append("backport/main") + labels.append(f"backport/{get_default_branch()}") updated_pr = github.update_pr( pull_number=pr.number, diff --git a/tasks/libs/ciproviders/gitlab_api.py b/tasks/libs/ciproviders/gitlab_api.py index 212e2ce38f142..0cf4c429b59b0 100644 --- a/tasks/libs/ciproviders/gitlab_api.py +++ b/tasks/libs/ciproviders/gitlab_api.py @@ -23,8 +23,7 @@ from invoke.exceptions import Exit from tasks.libs.common.color import Color, color_message -from tasks.libs.common.constants import DEFAULT_BRANCH -from tasks.libs.common.git import get_common_ancestor, get_current_branch +from tasks.libs.common.git import get_common_ancestor, get_current_branch, get_default_branch from tasks.libs.common.utils import retry_function BASE_URL = "https://gitlab.ddbuild.io" @@ -1214,7 +1213,7 @@ def compute_gitlab_ci_config_diff(ctx, before: str, after: str): after_name = after or "local files" # The before commit is the LCA commit between before and after - before = before or DEFAULT_BRANCH + before = before or get_default_branch() before = get_common_ancestor(ctx, before, after or "HEAD") print(f'Getting after changes config ({color_message(after_name, Color.BOLD)})') diff --git a/tasks/libs/common/constants.py b/tasks/libs/common/constants.py index ff9dc7a285a42..0b100a3432263 100644 --- a/tasks/libs/common/constants.py +++ b/tasks/libs/common/constants.py @@ -1,4 +1,3 @@ -DEFAULT_BRANCH = "main" DEFAULT_INTEGRATIONS_CORE_BRANCH = "master" GITHUB_ORG = "DataDog" REPO_NAME = "datadog-agent" diff --git a/tasks/libs/common/git.py b/tasks/libs/common/git.py index 8c7b9d9e2f080..6606a4ce522c2 100644 --- a/tasks/libs/common/git.py +++ b/tasks/libs/common/git.py @@ -5,10 +5,10 @@ from contextlib import contextmanager from typing import TYPE_CHECKING +from invoke import Context from invoke.exceptions import Exit from tasks.libs.common.color import Color, color_message -from tasks.libs.common.constants import DEFAULT_BRANCH from tasks.libs.common.user_interactions import yes_no_question if TYPE_CHECKING: @@ -90,7 +90,9 @@ def get_file_modifications( return modifications -def get_modified_files(ctx, base_branch="main") -> list[str]: +def get_modified_files(ctx, base_branch=None) -> list[str]: + base_branch = base_branch or get_default_branch() + return get_file_modifications( ctx, base_branch=base_branch, added=True, modified=True, only_names=True, no_renames=True ) @@ -100,7 +102,23 @@ def get_current_branch(ctx) -> str: return ctx.run("git rev-parse --abbrev-ref HEAD", hide=True).stdout.strip() -def get_common_ancestor(ctx, branch, base=DEFAULT_BRANCH) -> str: +def is_agent6(ctx) -> bool: + return get_current_branch(ctx).startswith("6.") + + +def get_default_branch(): + """Returns the default git branch given the current context (agent 6 / 7).""" + + # We create a context to avoid passing context in each function + # This context is used to get the current branch so there is no side effect + ctx = Context() + + return '6.53.x' if is_agent6(ctx) else 'main' + + +def get_common_ancestor(ctx, branch, base=None) -> str: + base = base or get_default_branch() + return ctx.run(f"git merge-base {branch} {base}", hide=True).stdout.strip() @@ -132,7 +150,7 @@ def get_main_parent_commit(ctx) -> str: """ Get the commit sha your current branch originated from """ - return ctx.run("git merge-base HEAD origin/main", hide=True).stdout.strip() + return ctx.run(f"git merge-base HEAD origin/{get_default_branch()}", hide=True).stdout.strip() def check_base_branch(branch, release_version): @@ -140,7 +158,7 @@ def check_base_branch(branch, release_version): Checks if the given branch is either the default branch or the release branch associated with the given release version. """ - return branch == DEFAULT_BRANCH or branch == release_version.branch() + return branch == get_default_branch() or branch == release_version.branch() def try_git_command(ctx, git_command): diff --git a/tasks/libs/common/gomodules.py b/tasks/libs/common/gomodules.py index ffbdd9f03d10d..357ab03dc515e 100644 --- a/tasks/libs/common/gomodules.py +++ b/tasks/libs/common/gomodules.py @@ -7,13 +7,13 @@ import sys from collections.abc import Callable from dataclasses import dataclass -from functools import lru_cache from pathlib import Path from typing import ClassVar import yaml import tasks +from tasks.libs.common.utils import agent_working_directory class ConfigDumper(yaml.SafeDumper): @@ -305,7 +305,6 @@ def dependency_path(self, agent_version): AGENT_MODULE_PATH_PREFIX = "github.com/DataDog/datadog-agent/" -@lru_cache def get_default_modules(base_dir: Path | None = None) -> dict[str, GoModule]: """Load the default modules from the modules.yml file. @@ -313,6 +312,8 @@ def get_default_modules(base_dir: Path | None = None) -> dict[str, GoModule]: base_dir: Root directory of the agent repository ('.' by default). """ + base_dir = base_dir or agent_working_directory() + return Configuration.from_file(base_dir).modules diff --git a/tasks/libs/common/utils.py b/tasks/libs/common/utils.py index 10882ddfeed7d..44758ae35b543 100644 --- a/tasks/libs/common/utils.py +++ b/tasks/libs/common/utils.py @@ -24,8 +24,8 @@ from invoke.exceptions import Exit from tasks.libs.common.color import Color, color_message -from tasks.libs.common.constants import ALLOWED_REPO_ALL_BRANCHES, DEFAULT_BRANCH, REPO_PATH -from tasks.libs.common.git import get_commit_sha +from tasks.libs.common.constants import ALLOWED_REPO_ALL_BRANCHES, REPO_PATH +from tasks.libs.common.git import get_commit_sha, get_default_branch from tasks.libs.owners.parsing import search_owners from tasks.libs.releasing.version import get_version from tasks.libs.types.arch import Arch @@ -373,6 +373,7 @@ def get_version_ldflags(ctx, major_version='7', install_path=None): Compute the version from the git tags, and set the appropriate compiler flags """ + payload_v = get_payload_version() commit = get_commit_sha(ctx, short=True) @@ -494,8 +495,8 @@ def environ(env): def is_pr_context(branch, pr_id, test_name): - if branch == DEFAULT_BRANCH: - print(f"Running on {DEFAULT_BRANCH}, skipping check for {test_name}.") + if branch == get_default_branch(): + print(f"Running on {get_default_branch()}, skipping check for {test_name}.") return False if not pr_id: print(f"PR not found, skipping check for {test_name}.") @@ -749,3 +750,11 @@ def get_metric_origin(origin_product, origin_sub_product, origin_product_detail, if origin_field: return {"origin": metric_origin} return metric_origin + + +def agent_working_directory(): + """Returns the working directory for the current context (agent 6 / 7).""" + + from tasks.libs.common.worktree import LOCAL_DIRECTORY, WORKTREE_DIRECTORY, is_worktree + + return WORKTREE_DIRECTORY if is_worktree() else LOCAL_DIRECTORY diff --git a/tasks/libs/common/worktree.py b/tasks/libs/common/worktree.py new file mode 100644 index 0000000000000..7e7da54af73df --- /dev/null +++ b/tasks/libs/common/worktree.py @@ -0,0 +1,109 @@ +"""Worktree utilities, used to execute tasks from this local repository (main) to a worktree with a different HEAD (e.g. 6.53.x). + +Common environment variables that can be used: +- WORKTREE_NO_PULL: If set to any value, the worktree will not be pulled before running the command. +""" + +import os +from contextlib import contextmanager +from pathlib import Path + +from invoke.exceptions import Exit + +from tasks.libs.common.color import Color, color_message +from tasks.libs.common.git import get_current_branch + +WORKTREE_DIRECTORY = Path.cwd().parent / "datadog-agent-worktree" +LOCAL_DIRECTORY = Path.cwd().resolve() + + +def init_env(ctx, branch: str | None = None): + """Will prepare the environment for commands applying to a worktree. + + To be used before each worktree section. + Will: + 1. Add the agent worktree if not present. + 2. Fetch the latest changes from the agent worktree. + """ + + if not WORKTREE_DIRECTORY.is_dir(): + if not ctx.run(f"git worktree add '{WORKTREE_DIRECTORY}' origin/{branch or 'main'}", warn=True): + raise Exit( + f'{color_message("Error", Color.RED)}: Cannot initialize worktree environment. You might want to reset the worktree directory with `inv worktree.remove`', + code=1, + ) + + if branch: + worktree_branch = ctx.run( + f"git -C '{WORKTREE_DIRECTORY}' rev-parse --abbrev-ref HEAD", hide=True + ).stdout.strip() + if worktree_branch != branch: + ctx.run(f"git -C '{WORKTREE_DIRECTORY}' checkout '{branch}'", hide=True) + + if not os.environ.get("AGENT_WORKTREE_NO_PULL"): + ctx.run(f"git -C '{WORKTREE_DIRECTORY}' pull", hide=True) + + +def remove_env(ctx): + """Will remove the environment for commands applying to a worktree.""" + + ctx.run(f"git worktree remove -f '{WORKTREE_DIRECTORY}'", warn=True) + + +def is_worktree(): + """Will return True if the current environment is a worktree environment.""" + + return Path.cwd() == WORKTREE_DIRECTORY + + +def enter_env(ctx, branch: str | None, skip_checkout=False): + """Enters the worktree environment.""" + + if not branch: + assert skip_checkout, 'skip_checkout must be set to True if branch is None' + + if not skip_checkout: + init_env(ctx, branch) + else: + assert WORKTREE_DIRECTORY.is_dir(), "Worktree directory is not present and skip_checkout is set to True" + + os.chdir(WORKTREE_DIRECTORY) + if skip_checkout and branch: + current_branch = get_current_branch(ctx) + assert ( + current_branch == branch + ), f"skip_checkout is True but the current branch ({current_branch}) is not {branch}. You should check out the branch before using this command, this can be safely done with `inv worktree.checkout {branch}`." + + +def exit_env(): + """Exits the worktree environment.""" + + os.chdir(LOCAL_DIRECTORY) + + +@contextmanager +def agent_context(ctx, branch: str | None, skip_checkout=False): + """Applies code to the worktree environment if the branch is not None. + + Args: + branch: The branch to switch to. If None, will enter the worktree environment without switching branch (ensures that skip_checkout is True). + skip_checkout: If True, the branch will not be checked out (no pull will be performed too). + + Usage: + > with agent_context(ctx, branch): + > ctx.run("head CHANGELOG.rst") # Displays the changelog of the target branch + """ + + # Do not stack two environments + if is_worktree(): + yield + return + + try: + # Enter + enter_env(ctx, branch, skip_checkout=skip_checkout) + + yield + finally: + # Exit + exit_env() diff --git a/tasks/libs/notify/pipeline_status.py b/tasks/libs/notify/pipeline_status.py index 25915ccc36d82..569973decfb0b 100644 --- a/tasks/libs/notify/pipeline_status.py +++ b/tasks/libs/notify/pipeline_status.py @@ -2,7 +2,7 @@ import re from tasks.libs.ciproviders.gitlab_api import get_commit, get_pipeline -from tasks.libs.common.constants import DEFAULT_BRANCH +from tasks.libs.common.git import get_default_branch from tasks.libs.notify.utils import DEPLOY_PIPELINES_CHANNEL, PIPELINES_CHANNEL, PROJECT_NAME from tasks.libs.pipeline.data import get_failed_jobs from tasks.libs.pipeline.notifications import ( @@ -40,7 +40,7 @@ def send_message(ctx, notification_type, dry_run): # For deploy pipelines not on the main branch, send notifications in a # dedicated channel. slack_channel = PIPELINES_CHANNEL - if notification_type == "deploy" and pipeline.ref != DEFAULT_BRANCH: + if notification_type == "deploy" and pipeline.ref != get_default_branch(): slack_channel = DEPLOY_PIPELINES_CHANNEL header = "" @@ -64,7 +64,7 @@ def send_message(ctx, notification_type, dry_run): else: send_slack_message(slack_channel, str(message)) - if should_send_message_to_author(pipeline.ref, DEFAULT_BRANCH): + if should_send_message_to_author(pipeline.ref, get_default_branch()): author_email = commit.author_email if dry_run: print(f"Would send to {author_email}:\n{str(message)}") diff --git a/tasks/libs/pipeline/tools.py b/tasks/libs/pipeline/tools.py index fc38ce67e883a..7f9d08e8eca6e 100644 --- a/tasks/libs/pipeline/tools.py +++ b/tasks/libs/pipeline/tools.py @@ -13,8 +13,8 @@ from tasks.libs.ciproviders.gitlab_api import refresh_pipeline from tasks.libs.common.color import Color, color_message +from tasks.libs.common.git import get_default_branch from tasks.libs.common.user_interactions import yes_no_question -from tasks.libs.common.utils import DEFAULT_BRANCH PIPELINE_FINISH_TIMEOUT_SEC = 3600 * 5 @@ -115,7 +115,7 @@ def gracefully_cancel_pipeline(repo: Project, pipeline: ProjectPipeline, force_c def trigger_agent_pipeline( repo: Project, - ref=DEFAULT_BRANCH, + ref=None, release_version_6="nightly", release_version_7="nightly-a7", branch="nightly", @@ -134,6 +134,8 @@ def trigger_agent_pipeline( - run a pipeline with all end-to-end tests, - run a deploy pipeline (includes all builds & kitchen tests + uploads artifacts to staging repositories); """ + + ref = ref or get_default_branch() args = {} if deploy: diff --git a/tasks/libs/releasing/json.py b/tasks/libs/releasing/json.py index e0ea089e43fa1..85270b23ce70e 100644 --- a/tasks/libs/releasing/json.py +++ b/tasks/libs/releasing/json.py @@ -7,6 +7,8 @@ from invoke.exceptions import Exit from tasks.libs.common.constants import TAG_FOUND_TEMPLATE +from tasks.libs.common.git import get_default_branch +from tasks.libs.common.worktree import is_worktree from tasks.libs.releasing.documentation import _stringify_config, nightly_entry_for, release_entry_for from tasks.libs.releasing.version import ( VERSION_RE, @@ -41,6 +43,12 @@ "datadog-agent-macos-build": "master", "datadog-agent": "main", } +DEFAULT_BRANCHES_AGENT6 = { + "omnibus-software": "6.53.x", + "omnibus-ruby": "6.53.x", + "datadog-agent-macos-build": "6.53.x", + "datadog-agent": "6.53.x", +} def load_release_json(): @@ -335,8 +343,12 @@ def generate_repo_data(warning_mode, next_version, release_branch): data = {} for repo in repos: branch = release_branch - if branch == "main": - branch = next_version.branch() if repo == "integrations-core" else DEFAULT_BRANCHES.get(repo, "main") + if branch == get_default_branch(): + branch = ( + next_version.branch() + if repo == "integrations-core" + else (DEFAULT_BRANCHES_AGENT6 if is_worktree() else DEFAULT_BRANCHES).get(repo, get_default_branch()) + ) data[repo] = { 'branch': branch, 'previous_tag': previous_tags.get(repo, ""), diff --git a/tasks/libs/releasing/notes.py b/tasks/libs/releasing/notes.py index 10edb3dab145d..d831f372b6197 100644 --- a/tasks/libs/releasing/notes.py +++ b/tasks/libs/releasing/notes.py @@ -2,7 +2,8 @@ from invoke import Failure -from tasks.libs.common.constants import DEFAULT_BRANCH, GITHUB_REPO_NAME +from tasks.libs.common.constants import GITHUB_REPO_NAME +from tasks.libs.common.git import get_default_branch from tasks.libs.releasing.version import current_version @@ -42,7 +43,7 @@ def _add_dca_prelude(ctx, agent7_version, agent6_version=""): f"""prelude: | Released on: {date.today()} - Pinned to datadog-agent v{agent7_version}: `CHANGELOG `_.""" + Pinned to datadog-agent v{agent7_version}: `CHANGELOG `_.""" ) ctx.run(f"git add {new_releasenote}") diff --git a/tasks/linter.py b/tasks/linter.py index 082c3065cc666..f549ae2005075 100644 --- a/tasks/linter.py +++ b/tasks/linter.py @@ -32,8 +32,8 @@ ) from tasks.libs.common.check_tools_version import check_tools_version from tasks.libs.common.color import Color, color_message -from tasks.libs.common.constants import DEFAULT_BRANCH, GITHUB_REPO_NAME -from tasks.libs.common.git import get_file_modifications, get_staged_files +from tasks.libs.common.constants import GITHUB_REPO_NAME +from tasks.libs.common.git import get_default_branch, get_file_modifications, get_staged_files from tasks.libs.common.utils import gitlab_section, is_pr_context, running_in_ci from tasks.libs.owners.parsing import read_owners from tasks.libs.types.copyright import CopyrightLinter, LintFailure @@ -52,7 +52,7 @@ def python(ctx): print( f"""Remember to set up pre-commit to lint your files before committing: - https://github.com/DataDog/datadog-agent/blob/{DEFAULT_BRANCH}/docs/dev/agent_dev_env.md#pre-commit-hooks""" + https://github.com/DataDog/datadog-agent/blob/{get_default_branch()}/docs/dev/agent_dev_env.md#pre-commit-hooks""" ) if running_in_ci(): diff --git a/tasks/modules.py b/tasks/modules.py index e26fe09b82b9f..43f91a2da959c 100644 --- a/tasks/modules.py +++ b/tasks/modules.py @@ -213,6 +213,7 @@ def validate_used_by_otel(ctx: Context): missing_used_by_otel_label: dict[str, list[str]] = defaultdict(list) # for every module labeled as "used_by_otel" + default_modules = get_default_modules() for otel_mod in otel_mods: gomod_path = f"{otel_mod}/go.mod" # get the go.mod data @@ -232,7 +233,7 @@ def validate_used_by_otel(ctx: Context): # we need the relative path of module (without github.com/DataDog/datadog-agent/ prefix) rel_path = require['Path'].removeprefix("github.com/DataDog/datadog-agent/") # check if indirect module is labeled as "used_by_otel" - if rel_path not in get_default_modules() or not get_default_modules()[rel_path].used_by_otel: + if rel_path not in default_modules or not default_modules[rel_path].used_by_otel: missing_used_by_otel_label[rel_path].append(otel_mod) if missing_used_by_otel_label: message = f"{color_message('ERROR', Color.RED)}: some indirect local dependencies of modules labeled \"used_by_otel\" are not correctly labeled in get_default_modules()\n" @@ -260,6 +261,7 @@ def show(_, path: str, remove_defaults: bool = False, base_dir: str = '.'): Args: remove_defaults: If True, will remove default values from the output. + base_dir: Where to load modules from. """ config = Configuration.from_file(Path(base_dir)) @@ -281,6 +283,7 @@ def show_all(_, base_dir: str = '.', ignored=False): """Show the list of modules. Args: + base_dir: Where to load modules from. ignored: If True, will list ignored modules. """ diff --git a/tasks/pipeline.py b/tasks/pipeline.py index 31b728798c334..8d63109fee536 100644 --- a/tasks/pipeline.py +++ b/tasks/pipeline.py @@ -19,8 +19,7 @@ refresh_pipeline, ) from tasks.libs.common.color import Color, color_message -from tasks.libs.common.constants import DEFAULT_BRANCH -from tasks.libs.common.git import get_commit_sha, get_current_branch +from tasks.libs.common.git import get_commit_sha, get_current_branch, get_default_branch from tasks.libs.common.utils import ( get_all_allowed_repo_branches, is_allowed_repo_branch, @@ -94,7 +93,7 @@ def check_deploy_pipeline(repo: Project, git_ref: str, release_version_6, releas @task -def clean_running_pipelines(ctx, git_ref=DEFAULT_BRANCH, here=False, use_latest_sha=False, sha=None): +def clean_running_pipelines(ctx, git_ref=None, here=False, use_latest_sha=False, sha=None): """ Fetch running pipelines on a target ref (+ optionally a git sha), and ask the user if they should be cancelled. @@ -104,6 +103,8 @@ def clean_running_pipelines(ctx, git_ref=DEFAULT_BRANCH, here=False, use_latest_ if here: git_ref = get_current_branch(ctx) + else: + git_ref = git_ref or get_default_branch() print(f"Fetching running pipelines on {git_ref}") @@ -130,11 +131,12 @@ def workflow_rules(gitlab_file=".gitlab-ci.yml"): @task -def trigger(_, git_ref=DEFAULT_BRANCH, release_version_6="dev", release_version_7="dev-a7", repo_branch="dev"): +def trigger(_, git_ref=None, release_version_6="dev", release_version_7="dev-a7", repo_branch="dev"): """ OBSOLETE: Trigger a deploy pipeline on the given git ref. Use pipeline.run with the --deploy option instead. """ + git_ref = git_ref or get_default_branch() use_release_entries = "" major_versions = [] @@ -829,16 +831,16 @@ def test_merge_queue(ctx): # Create a new main and push it print("Creating a new main branch") timestamp = int(datetime.now(timezone.utc).timestamp()) - test_main = f"mq/test_{timestamp}" + test_default = f"mq/test_{timestamp}" current_branch = get_current_branch(ctx) - ctx.run("git checkout main", hide=True) + ctx.run(f"git checkout {get_default_branch()}", hide=True) ctx.run("git pull", hide=True) - ctx.run(f"git checkout -b {test_main}", hide=True) - ctx.run(f"git push origin {test_main}", hide=True) + ctx.run(f"git checkout -b {test_default}", hide=True) + ctx.run(f"git push origin {test_default}", hide=True) # Create a PR towards this new branch and adds it to the merge queue print("Creating a PR and adding it to the merge queue") gh = GithubAPI() - pr = gh.create_pr(f"Test MQ for {current_branch}", "", test_main, current_branch) + pr = gh.create_pr(f"Test MQ for {current_branch}", "", test_default, current_branch) pr.create_issue_comment("/merge") # Search for the generated pipeline print(f"PR {pr.html_url} is waiting for MQ pipeline generation") @@ -848,7 +850,7 @@ def test_merge_queue(ctx): time.sleep(30) pipelines = agent.pipelines.list(per_page=100) try: - pipeline = next(p for p in pipelines if p.ref.startswith(f"mq-working-branch-{test_main}")) + pipeline = next(p for p in pipelines if p.ref.startswith(f"mq-working-branch-{test_default}")) print(f"Pipeline found: {pipeline.web_url}") break except StopIteration as e: @@ -866,8 +868,8 @@ def test_merge_queue(ctx): pipeline.cancel() pr.edit(state="closed") ctx.run(f"git checkout {current_branch}", hide=True) - ctx.run(f"git branch -D {test_main}", hide=True) - ctx.run(f"git push origin :{test_main}", hide=True) + ctx.run(f"git branch -D {test_default}", hide=True) + ctx.run(f"git push origin :{test_default}", hide=True) if not success: raise Exit(message="Merge queue test failed", code=1) diff --git a/tasks/release.py b/tasks/release.py index a82deec90161f..5e0b11483a938 100644 --- a/tasks/release.py +++ b/tasks/release.py @@ -20,7 +20,6 @@ from tasks.libs.ciproviders.gitlab_api import get_gitlab_repo from tasks.libs.common.color import Color, color_message from tasks.libs.common.constants import ( - DEFAULT_BRANCH, GITHUB_REPO_NAME, ) from tasks.libs.common.git import ( @@ -28,8 +27,10 @@ check_clean_branch_state, clone, get_current_branch, + get_default_branch, get_last_commit, get_last_release_tag, + is_agent6, try_git_command, ) from tasks.libs.common.gomodules import get_default_modules @@ -47,6 +48,8 @@ release_manager, ) from tasks.libs.releasing.json import ( + DEFAULT_BRANCHES, + DEFAULT_BRANCHES_AGENT6, UNFREEZE_REPO_AGENT, UNFREEZE_REPOS, _get_release_json_value, @@ -371,7 +374,7 @@ def create_rc(ctx, major_versions="6,7", patch_version=False, upstream="origin", if not check_base_branch(current_branch, new_highest_version): raise Exit( color_message( - f"The branch you are on is neither {DEFAULT_BRANCH} or the correct release branch ({new_highest_version.branch()}). Aborting.", + f"The branch you are on is neither {get_default_branch()} or the correct release branch ({new_highest_version.branch()}). Aborting.", "red", ), code=1, @@ -472,7 +475,7 @@ def build_rc(ctx, major_versions="6,7", patch_version=False, k8s_deployments=Fal if not check_base_branch(current_branch, new_version): raise Exit( color_message( - f"The branch you are on is neither {DEFAULT_BRANCH} or the correct release branch ({new_version.branch()}). Aborting.", + f"The branch you are on is neither {get_default_branch()} or the correct release branch ({new_version.branch()}). Aborting.", "red", ), code=1, @@ -663,7 +666,7 @@ def create_release_branches(ctx, base_directory="~/dd", major_versions="6,7", up create_release_pr( f"[release] Update current milestone to {next}", - "main", + get_default_branch(), milestone_branch, next, ) @@ -683,8 +686,8 @@ def create_release_branches(ctx, base_directory="~/dd", major_versions="6,7", up with open(file, "w") as gl: for line in file_content: - if re.search(r"compare_to: main", line): - gl.write(line.replace("main", f"{release_branch}")) + if re.search(rf"compare_to: {get_default_branch()}", line): + gl.write(line.replace(get_default_branch(), f"{release_branch}")) else: gl.write(line) @@ -754,7 +757,7 @@ def cleanup(ctx): current_milestone = _update_last_stable(ctx, version) # create pull request to update last stable version - main_branch = "main" + main_branch = get_default_branch() cleanup_branch = f"release/{version}-cleanup" ctx.run(f"git checkout -b {cleanup_branch}") ctx.run("git add release.json") @@ -787,9 +790,10 @@ def cleanup(ctx): @task def check_omnibus_branches(ctx): base_branch = _get_release_json_value('base_branch') - if base_branch == 'main': - omnibus_ruby_branch = 'datadog-5.5.0' - omnibus_software_branch = 'master' + if base_branch == get_default_branch(): + default_branches = DEFAULT_BRANCHES_AGENT6 if is_agent6(ctx) else DEFAULT_BRANCHES + omnibus_ruby_branch = default_branches['omnibus-ruby'] + omnibus_software_branch = default_branches['omnibus-software'] else: omnibus_ruby_branch = base_branch omnibus_software_branch = base_branch @@ -913,7 +917,7 @@ def get_active_release_branch(_): if release_branch: print(f"{release_branch.name}") else: - print("main") + print(get_default_branch()) @task diff --git a/tasks/setup.py b/tasks/setup.py index 0426543a16010..5e1eeee98d1f1 100644 --- a/tasks/setup.py +++ b/tasks/setup.py @@ -17,6 +17,7 @@ from tasks import vscode from tasks.libs.common.color import Color, color_message +from tasks.libs.common.git import get_default_branch from tasks.libs.common.status import Status from tasks.libs.common.utils import running_in_pyapp @@ -95,7 +96,7 @@ def check_git_repo(ctx) -> SetupResult: ctx.run("git fetch", hide=True) print(color_message("Checking main branch...", Color.BLUE)) - output = ctx.run('git rev-list "^HEAD" origin/main --count', hide=True) + output = ctx.run(f'git rev-list "^HEAD" origin/{get_default_branch()} --count', hide=True) count = output.stdout.strip() message = "" @@ -103,7 +104,7 @@ def check_git_repo(ctx) -> SetupResult: if count != "0": status = Status.WARN - message = f"Your branch is {count} commit(s) behind main. Please update your branch." + message = f"Your branch is {count} commit(s) behind {get_default_branch()}. Please update your branch." return SetupResult("Check git repository", status, message) diff --git a/tasks/unit_tests/libs/common/worktree_tests.py b/tasks/unit_tests/libs/common/worktree_tests.py new file mode 100644 index 0000000000000..3998357c1accc --- /dev/null +++ b/tasks/unit_tests/libs/common/worktree_tests.py @@ -0,0 +1,97 @@ +import os +import unittest + +from invoke import Context + +from tasks.libs.common.git import get_default_branch +from tasks.libs.common.gomodules import get_default_modules +from tasks.libs.common.worktree import agent_context, init_env, is_worktree + + +def get_ctx(): + return Context() + + +class TestWorktree(unittest.TestCase): + def setUp(self): + # Pull only once + init_env(get_ctx(), '6.53.x') + os.environ['AGENT_WORKTREE_NO_PULL'] = '1' + + def test_context_is_worktree_true(self): + with agent_context(get_ctx(), '6.53.x'): + self.assertTrue(is_worktree()) + + def test_context_is_worktree_false(self): + self.assertFalse(is_worktree()) + + def test_context_nested(self): + with agent_context(get_ctx(), '6.53.x'): + with agent_context(get_ctx(), '6.53.x'): + self.assertTrue(is_worktree()) + self.assertTrue(is_worktree()) + + def test_context_pwd(self): + ctx = get_ctx() + + with agent_context(ctx, None, skip_checkout=True): + pwdnone = ctx.run('pwd').stdout + + with agent_context(ctx, '6.53.x'): + pwd6 = ctx.run('pwd').stdout + + with agent_context(ctx, 'main'): + pwdmain = ctx.run('pwd').stdout + + self.assertEqual(pwd6, pwdnone) + self.assertEqual(pwd6, pwdmain) + + def test_context_modules(self): + ctx = get_ctx() + + with agent_context(ctx, 'main'): + modules7 = get_default_modules() + + with agent_context(ctx, '6.53.x'): + modules6 = get_default_modules() + + self.assertNotEqual(set(modules6.keys()), set(modules7.keys())) + + def test_context_branch(self): + ctx = get_ctx() + + with agent_context(ctx, 'main'): + branch7 = get_default_branch() + + with agent_context(ctx, '6.53.x'): + branch6 = get_default_branch() + + self.assertNotEqual(branch6, branch7) + + def test_context_no_checkout(self): + ctx = get_ctx() + + with agent_context(ctx, '6.53.x'): + branch6 = get_default_branch() + + with agent_context(ctx, 'main'): + branch7 = get_default_branch() + + with agent_context(ctx, 'main', skip_checkout=True): + branch_no_checkout = get_default_branch() + + self.assertNotEqual(branch6, branch7) + self.assertEqual(branch7, branch_no_checkout) + + def test_context_no_checkout_error(self): + ctx = get_ctx() + + with agent_context(ctx, '6.53.x'): + pass + + def switch_context(): + # The current branch is not main + with agent_context(ctx, 'main', skip_checkout=True): + pass + + self.assertRaises(AssertionError, switch_context) diff --git a/tasks/worktree.py b/tasks/worktree.py new file mode 100644 index 0000000000000..bb00ed751b370 --- /dev/null +++ b/tasks/worktree.py @@ -0,0 +1,88 @@ +from invoke import task +from invoke.exceptions import Exit + +from tasks.libs.common.color import Color, color_message +from tasks.libs.common.user_interactions import yes_no_question +from tasks.libs.common.worktree import WORKTREE_DIRECTORY, agent_context, enter_env, init_env, remove_env + + +@task +def init(ctx, branch: str | None = None): + """Will prepare the worktree context (git clone / pull of the agent branch).""" + + init_env(ctx, branch) + + +@task +def remove(ctx): + """Will remove the git worktree context.""" + + remove_env(ctx) + + +@task +def status(ctx): + """Displays the status of the worktree environment.""" + + if not WORKTREE_DIRECTORY.is_dir(): + raise Exit('No worktree environment found.') + + ctx.run(f"git -C '{WORKTREE_DIRECTORY}' status", pty=True) + + +@task +def checkout(ctx, ref): + """Changes the worktree environment to the specified ref. + + Note: + This won't pull. + """ + + if not WORKTREE_DIRECTORY.is_dir(): + raise Exit('No worktree environment found.') + + ctx.run(f"git -C '{WORKTREE_DIRECTORY}' checkout '{ref}'", pty=True) + + +@task +def pull(ctx): + """Pulls the worktree environment.""" + + if not WORKTREE_DIRECTORY.is_dir(): + raise Exit('No worktree environment found.') + + ctx.run(f"git -C '{WORKTREE_DIRECTORY}' pull", pty=True) + + +@task +def run(ctx, branch: str, command: str, skip_checkout: bool = False): + """Runs a command in the target worktree environment. + + Usage: + $ inv worktree.run 6.53.x "head CHANGELOG.rst" # Displays the changelog of the target branch + """ + + with agent_context(ctx, branch, skip_checkout=skip_checkout): + ctx.run(command) + + +@task +def invoke(ctx, branch: str, skip_checkout: bool = False, yes: bool = False): + """Enters the worktree environment in order to invoke tasks in this context. + + Note: + This task should be avoided when a --branch or --release-branch argument is available in the task. + + Usage: + > inv worktree.invoke 6.53.x --yes modules.show-all # Will show agent 6 modules + """ + + if yes or yes_no_question( + 'Warning: This task should be avoided, use --branch or --release-branch argument if available in the task. Want to proceed?', + color=Color.ORANGE, + default=False, + ): + # The tasks running after this one will be using the agent 6 environment + enter_env(ctx, branch, skip_checkout=skip_checkout) + else: + raise Exit(color_message('Aborted.', Color.RED))