From 5cf320ce3ec3db62f1a7ed479923951c5feecbb4 Mon Sep 17 00:00:00 2001 From: Andrew Halberstadt Date: Tue, 31 Mar 2026 10:14:47 -0400 Subject: [PATCH 1/2] feat(run-task): add ability to fetch arbitrary extra refs Bug: 2028208 --- src/taskgraph/run-task/run-task | 10 ++++++- test/test_scripts_run_task.py | 46 +++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/src/taskgraph/run-task/run-task b/src/taskgraph/run-task/run-task index deca3a691..8f85006e8 100755 --- a/src/taskgraph/run-task/run-task +++ b/src/taskgraph/run-task/run-task @@ -31,7 +31,7 @@ import time import urllib.error import urllib.request from pathlib import Path -from typing import Dict, Optional +from typing import Dict, List, Optional SECRET_BASEURL_TPL = "{}/secrets/v1/secret/{{}}".format( os.environ.get("TASKCLUSTER_PROXY_URL", "http://taskcluster").rstrip("/") @@ -640,6 +640,7 @@ def git_checkout( ssh_key_file: Optional[Path], ssh_known_hosts_file: Optional[Path], shallow: bool = False, + extra_refs: Optional[List[str]] = None, ): assert head_ref or head_rev @@ -734,6 +735,9 @@ def git_checkout( env=env, ) + for ref in extra_refs or []: + git_fetch(destination_path, f"{ref}:{ref}", remote=head_repo, env=env) + args = [ "git", "checkout", @@ -932,6 +936,8 @@ def collect_vcs_options(args, project, name): head_rev = os.environ.get(f"{env_prefix}_HEAD_REV") pip_requirements = os.environ.get(f"{env_prefix}_PIP_REQUIREMENTS") private_key_secret = os.environ.get(f"{env_prefix}_SSH_SECRET_NAME") + if extra_refs := os.environ.get(f"{env_prefix}_EXTRA_REFS"): + extra_refs = json.loads(extra_refs) store_path = os.environ.get("HG_STORE_PATH") @@ -965,6 +971,7 @@ def collect_vcs_options(args, project, name): "ssh-secret-name": private_key_secret, "pip-requirements": pip_requirements, "shallow-clone": shallow_clone, + "extra-refs": extra_refs, } @@ -1014,6 +1021,7 @@ def vcs_checkout_from_args(options): ssh_key_file, ssh_known_hosts_file, shallow=options.get("shallow-clone", False), + extra_refs=options.get("extra-refs"), ) elif options["repo-type"] == "hg": revision = hg_checkout( diff --git a/test/test_scripts_run_task.py b/test/test_scripts_run_task.py index b2393721c..0d60a8d41 100644 --- a/test/test_scripts_run_task.py +++ b/test/test_scripts_run_task.py @@ -1,4 +1,5 @@ import functools +import json import os import site import stat @@ -179,6 +180,17 @@ def test_install_pip_requirements_with_uv( {"shallow-clone": True}, id="git_with_shallow_clone", ), + pytest.param( + {}, + { + "REPOSITORY_TYPE": "git", + "HEAD_REPOSITORY": "https://github.com/example/repo", + "HEAD_REV": "abc123", + "EXTRA_REFS": json.dumps(["refs/notes/taskgraph", "refs/notes/other"]), + }, + {"extra-refs": ["refs/notes/taskgraph", "refs/notes/other"]}, + id="git_with_extra_refs", + ), ], ) def test_collect_vcs_options( @@ -216,6 +228,7 @@ def test_collect_vcs_options( "shallow-clone": False, "ssh-secret-name": env.get("SSH_SECRET_NAME"), "store-path": env.get("HG_STORE_PATH"), + "extra-refs": None, } if "PIP_REQUIREMENTS" in env: expected["pip-requirements"] = os.path.join( @@ -638,3 +651,36 @@ def test_main_abspath_environment(mocker, run_main): assert env.get("MOZ_UV_HOME") == "/builds/worker/dir/uv" for key in envvars: assert env[key] == "/builds/worker/file" + + +def test_git_checkout_extra_refs(mock_stdin, run_task_mod, mock_git_repo, tmp_path): + """extra_refs are fetched into the local repo during checkout.""" + # Add a notes ref to the source repo + rev = mock_git_repo["main"][-1] + subprocess.check_call( + ["git", "notes", "--ref=refs/notes/taskgraph", "add", "-m", "test", rev], + cwd=mock_git_repo["path"], + ) + + destination = tmp_path / "destination" + run_task_mod.git_checkout( + destination_path=str(destination), + head_repo=mock_git_repo["path"], + base_repo=mock_git_repo["path"], + base_rev=None, + head_ref="main", + head_rev=None, + ssh_key_file=None, + ssh_known_hosts_file=None, + extra_refs=["refs/notes/taskgraph"], + ) + + # Verify the notes ref is available locally + result = subprocess.run( + ["git", "notes", "--ref=refs/notes/taskgraph", "show", rev], + cwd=str(destination), + capture_output=True, + text=True, + ) + assert result.returncode == 0 + assert "test" in result.stdout From d8a1a6350b256c11c598f85a80c10e74db5183e6 Mon Sep 17 00:00:00 2001 From: Andrew Halberstadt Date: Tue, 31 Mar 2026 10:51:25 -0400 Subject: [PATCH 2/2] feat(decision): support extracting parameters from Git note Bug: 2028208 --- .taskcluster.yml | 7 ++++- src/taskgraph/decision.py | 14 ++++++++++ src/taskgraph/main.py | 6 ++++ src/taskgraph/util/vcs.py | 20 +++++++++++++ test/test_decision.py | 59 +++++++++++++++++++++++++++++++++++++++ test/test_util_vcs.py | 21 ++++++++++++++ 6 files changed, 126 insertions(+), 1 deletion(-) diff --git a/.taskcluster.yml b/.taskcluster.yml index 618705f61..a7ebf8754 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -226,6 +226,7 @@ tasks: - $if: 'isPullRequest' then: TASKGRAPH_PULL_REQUEST_NUMBER: '${event.pull_request.number}' + TASKGRAPH_EXTRA_REFS: {$json: ["refs/notes/decision-parameters"]} - $if: 'tasks_for == "action" || tasks_for == "pr-action"' then: ACTION_TASK_GROUP_ID: '${action.taskGroupId}' # taskGroupId of the target task @@ -251,7 +252,11 @@ tasks: - bash - -cx - $let: - extraArgs: {$if: 'tasks_for == "cron"', then: '${cron.quoted_args}', else: ''} + extraArgs: + $switch: + 'tasks_for == "cron"': '${cron.quoted_args}' + 'tasks_for == "github-pull-request"': '--allow-parameter-override' + $default: '' in: $if: 'tasks_for == "action" || tasks_for == "pr-action"' then: > diff --git a/src/taskgraph/decision.py b/src/taskgraph/decision.py index 35193f06e..b5ebb9e7c 100644 --- a/src/taskgraph/decision.py +++ b/src/taskgraph/decision.py @@ -264,6 +264,20 @@ def get_decision_parameters(graph_config, options): ] == "github-pull-request": set_try_config(parameters, task_config_file) + # load extra parameters from vcs note if able + note_ref = "refs/notes/decision-parameters" + if options.get("allow_parameter_override") and ( + note_params := repo.get_note(note_ref) + ): + try: + note_params = json.loads(note_params) + logger.info( + f"Overriding parameters from {note_ref}:\n{json.dumps(note_params, indent=2)}" + ) + parameters.update(note_params) + except ValueError as e: + raise Exception(f"Failed to parse {note_ref} as JSON: {e}") from e + result = Parameters(**parameters) result.check() return result diff --git a/src/taskgraph/main.py b/src/taskgraph/main.py index 18a3633db..5b72d9201 100644 --- a/src/taskgraph/main.py +++ b/src/taskgraph/main.py @@ -933,6 +933,12 @@ def load_task(args): @argument( "--verbose", "-v", action="store_true", help="include debug-level logging output" ) +@argument( + "--allow-parameter-override", + default=False, + action="store_true", + help="Allow user to override computed decision task parameters.", +) def decision(options): from taskgraph.decision import taskgraph_decision # noqa: PLC0415 diff --git a/src/taskgraph/util/vcs.py b/src/taskgraph/util/vcs.py index 602cb81c5..584b166f9 100644 --- a/src/taskgraph/util/vcs.py +++ b/src/taskgraph/util/vcs.py @@ -215,6 +215,14 @@ def does_revision_exist_locally(self, revision: str) -> bool: If this function returns an unexpected value, then make sure the revision was fetched from the remote repository.""" + def get_note(self, note: str, revision: Optional[str] = None) -> Optional[str]: + """Read a note attached to the given revision (defaults to HEAD). + + Returns the note content as a string, or ``None`` if no note exists. + Only supported by Git; returns ``None`` for all other VCS types. + """ + return None + class HgRepository(Repository): @property @@ -586,6 +594,18 @@ def does_revision_exist_locally(self, revision): return False raise + def get_note(self, note: str, revision: Optional[str] = None) -> Optional[str]: + if not note.startswith("refs/notes/"): + note = f"refs/notes/{note}" + + revision = revision or "HEAD" + try: + return self.run("notes", f"--ref={note}", "show", revision).strip() + except subprocess.CalledProcessError as e: + if e.returncode == 1: + return None + raise + def get_repository(path: str): """Get a repository object for the repository at `path`. diff --git a/test/test_decision.py b/test/test_decision.py index f3071a938..2d9b23baa 100644 --- a/test/test_decision.py +++ b/test/test_decision.py @@ -10,6 +10,8 @@ import unittest from pathlib import Path +import pytest + from taskgraph import decision from taskgraph.util.vcs import GitRepository, HgRepository from taskgraph.util.yaml import load_yaml @@ -137,3 +139,60 @@ def test_dontbuild_commit_message_yields_default_target_tasks_method( self.options["tasks_for"] = "hg-push" params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options) self.assertEqual(params["target_tasks_method"], "nothing") + + +_BASE_OPTIONS = { + "base_repository": "https://hg.mozilla.org/mozilla-unified", + "base_rev": "aaaa", + "head_repository": "https://hg.mozilla.org/mozilla-central", + "head_rev": "bbbb", + "head_ref": "default", + "head_tag": "", + "project": "mozilla-central", + "pushlog_id": "1", + "pushdate": 0, + "repository_type": "git", + "owner": "nobody@mozilla.com", + "tasks_for": "github-push", + "level": "1", +} + + +@unittest.mock.patch.object( + GitRepository, + "get_note", + return_value=json.dumps({"build_number": 99}), +) +@unittest.mock.patch.object(GitRepository, "get_changed_files", return_value=[]) +def test_decision_parameters_note(mock_files_changed, mock_get_note): + options = {**_BASE_OPTIONS, "allow_parameter_override": True} + params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, options) + mock_get_note.assert_called_once_with("refs/notes/decision-parameters") + assert params["build_number"] == 99 + + +@unittest.mock.patch.object( + GitRepository, + "get_note", + return_value=json.dumps({"build_number": 99}), +) +@unittest.mock.patch.object(GitRepository, "get_changed_files", return_value=[]) +def test_decision_parameters_note_disallow_override(mock_files_changed, mock_get_note): + options = {**_BASE_OPTIONS, "allow_parameter_override": False} + params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, options) + mock_get_note.assert_not_called() + assert params["build_number"] == 1 + + +@unittest.mock.patch.object( + GitRepository, + "get_note", + return_value="not valid json {", +) +@unittest.mock.patch.object(GitRepository, "get_changed_files", return_value=[]) +def test_decision_parameters_note_invalid_json(mock_files_changed, mock_get_note): + options = {**_BASE_OPTIONS, "allow_parameter_override": True} + with pytest.raises( + Exception, match="Failed to parse refs/notes/decision-parameters as JSON" + ): + decision.get_decision_parameters(FAKE_GRAPH_CONFIG, options) diff --git a/test/test_util_vcs.py b/test/test_util_vcs.py index da710307e..84b97940d 100644 --- a/test/test_util_vcs.py +++ b/test/test_util_vcs.py @@ -3,6 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. import os +import shutil import subprocess from pathlib import Path from textwrap import dedent @@ -648,3 +649,23 @@ def test_get_changed_files_with_null_base_revision_shallow_clone( assert "first_file" in changed_files assert "file1.txt" in changed_files assert "file2.txt" in changed_files + + +def test_get_note_git(git_repo, tmpdir): + """get_note returns note content when present, None otherwise.""" + repo_path = tmpdir.join("git") + shutil.copytree(git_repo, repo_path) + repo = get_repository(str(repo_path)) + + # No note yet + assert repo.get_note("try-config") is None + + rev = repo.head_rev + subprocess.check_call( + ["git", "notes", "--ref=refs/notes/try-config", "add", "-m", "test note", rev], + cwd=repo.path, + ) + + assert repo.get_note("try-config") == "test note" + assert repo.get_note("try-config", rev) == "test note" + assert repo.get_note("other") is None