From 147a0af5e8bb25bb7ed8de12f5348cb13d8cccf6 Mon Sep 17 00:00:00 2001 From: Anas Seth Date: Sat, 4 Apr 2026 07:51:58 +0500 Subject: [PATCH] feat(extensions): authenticate GitHub-hosted catalog and download requests with GITHUB_TOKEN/GH_TOKEN --- extensions/EXTENSION-USER-GUIDE.md | 17 +++- src/specify_cli/extensions.py | 27 ++++++- tests/test_extensions.py | 126 +++++++++++++++++++++++++++++ 3 files changed, 166 insertions(+), 4 deletions(-) diff --git a/extensions/EXTENSION-USER-GUIDE.md b/extensions/EXTENSION-USER-GUIDE.md index 190e263af2..ca8adfbaab 100644 --- a/extensions/EXTENSION-USER-GUIDE.md +++ b/extensions/EXTENSION-USER-GUIDE.md @@ -421,7 +421,7 @@ In addition to extension-specific environment variables (`SPECKIT_{EXT_ID}_*`), | Variable | Description | Default | |----------|-------------|---------| | `SPECKIT_CATALOG_URL` | Override the full catalog stack with a single URL (backward compat) | Built-in default stack | -| `GH_TOKEN` / `GITHUB_TOKEN` | GitHub API token for downloads | None | +| `GH_TOKEN` / `GITHUB_TOKEN` | GitHub token for authenticated requests to GitHub-hosted URLs (`raw.githubusercontent.com`, `github.com`, `api.github.com`). Required when your catalog JSON or extension ZIPs are hosted in a private GitHub repository. | None | #### Example: Using a custom catalog for testing @@ -433,6 +433,21 @@ export SPECKIT_CATALOG_URL="http://localhost:8000/catalog.json" export SPECKIT_CATALOG_URL="https://example.com/staging/catalog.json" ``` +#### Example: Using a private GitHub-hosted catalog + +```bash +# Authenticate with a token (gh CLI, PAT, or GITHUB_TOKEN in CI) +export GITHUB_TOKEN=$(gh auth token) + +# Search a private catalog added via `specify extension catalog add` +specify extension search jira + +# Install from a private catalog +specify extension add jira-sync +``` + +The token is attached automatically to requests targeting GitHub domains. Non-GitHub catalog URLs are always fetched without credentials. + --- ## Extension Catalogs diff --git a/src/specify_cli/extensions.py b/src/specify_cli/extensions.py index 3420a7651b..096721ddae 100644 --- a/src/specify_cli/extensions.py +++ b/src/specify_cli/extensions.py @@ -1411,6 +1411,27 @@ def _validate_catalog_url(self, url: str) -> None: if not parsed.netloc: raise ValidationError("Catalog URL must be a valid URL with a host.") + def _make_request(self, url: str) -> "urllib.request.Request": + """Build a urllib Request, adding a GitHub auth header when available. + + Reads GITHUB_TOKEN or GH_TOKEN from the environment and attaches an + ``Authorization: token `` header for requests to GitHub-hosted + domains (``raw.githubusercontent.com``, ``github.com``, + ``api.github.com``). Non-GitHub URLs are returned as plain requests + so credentials are never leaked to third-party hosts. + """ + import os + import urllib.request + + headers: Dict[str, str] = {} + token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") + if token and any( + host in url + for host in ("raw.githubusercontent.com", "github.com", "api.github.com") + ): + headers["Authorization"] = f"token {token}" + return urllib.request.Request(url, headers=headers) + def _load_catalog_config(self, config_path: Path) -> Optional[List[CatalogEntry]]: """Load catalog stack configuration from a YAML file. @@ -1601,7 +1622,7 @@ def _fetch_single_catalog(self, entry: CatalogEntry, force_refresh: bool = False # Fetch from network try: - with urllib.request.urlopen(entry.url, timeout=10) as response: + with urllib.request.urlopen(self._make_request(entry.url), timeout=10) as response: catalog_data = json.loads(response.read()) if "schema_version" not in catalog_data or "extensions" not in catalog_data: @@ -1718,7 +1739,7 @@ def fetch_catalog(self, force_refresh: bool = False) -> Dict[str, Any]: import urllib.request import urllib.error - with urllib.request.urlopen(catalog_url, timeout=10) as response: + with urllib.request.urlopen(self._make_request(catalog_url), timeout=10) as response: catalog_data = json.loads(response.read()) # Validate catalog structure @@ -1861,7 +1882,7 @@ def download_extension(self, extension_id: str, target_dir: Optional[Path] = Non # Download the ZIP file try: - with urllib.request.urlopen(download_url, timeout=60) as response: + with urllib.request.urlopen(self._make_request(download_url), timeout=60) as response: zip_data = response.read() zip_path.write_bytes(zip_data) diff --git a/tests/test_extensions.py b/tests/test_extensions.py index 350b368eac..039dfc1f29 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -2142,6 +2142,132 @@ def test_clear_cache(self, temp_dir): assert not catalog.cache_file.exists() assert not catalog.cache_metadata_file.exists() + # --- _make_request / GitHub auth --- + + def _make_catalog(self, temp_dir): + project_dir = temp_dir / "project" + project_dir.mkdir() + (project_dir / ".specify").mkdir() + return ExtensionCatalog(project_dir) + + def test_make_request_no_token_no_auth_header(self, temp_dir, monkeypatch): + """Without a token, requests carry no Authorization header.""" + monkeypatch.delenv("GITHUB_TOKEN", raising=False) + monkeypatch.delenv("GH_TOKEN", raising=False) + catalog = self._make_catalog(temp_dir) + req = catalog._make_request("https://raw.githubusercontent.com/org/repo/main/catalog.json") + assert "Authorization" not in req.headers + + def test_make_request_github_token_added_for_raw_githubusercontent(self, temp_dir, monkeypatch): + """GITHUB_TOKEN is attached for raw.githubusercontent.com URLs.""" + monkeypatch.setenv("GITHUB_TOKEN", "ghp_testtoken") + monkeypatch.delenv("GH_TOKEN", raising=False) + catalog = self._make_catalog(temp_dir) + req = catalog._make_request("https://raw.githubusercontent.com/org/repo/main/catalog.json") + assert req.get_header("Authorization") == "token ghp_testtoken" + + def test_make_request_gh_token_fallback(self, temp_dir, monkeypatch): + """GH_TOKEN is used when GITHUB_TOKEN is absent.""" + monkeypatch.delenv("GITHUB_TOKEN", raising=False) + monkeypatch.setenv("GH_TOKEN", "ghp_ghtoken") + catalog = self._make_catalog(temp_dir) + req = catalog._make_request("https://github.com/org/repo/releases/download/v1/ext.zip") + assert req.get_header("Authorization") == "token ghp_ghtoken" + + def test_make_request_github_token_takes_precedence_over_gh_token(self, temp_dir, monkeypatch): + """GITHUB_TOKEN takes precedence over GH_TOKEN when both are set.""" + monkeypatch.setenv("GITHUB_TOKEN", "ghp_primary") + monkeypatch.setenv("GH_TOKEN", "ghp_secondary") + catalog = self._make_catalog(temp_dir) + req = catalog._make_request("https://api.github.com/repos/org/repo") + assert req.get_header("Authorization") == "token ghp_primary" + + def test_make_request_token_not_added_for_non_github_url(self, temp_dir, monkeypatch): + """Auth header is never attached to non-GitHub URLs to prevent credential leakage.""" + monkeypatch.setenv("GITHUB_TOKEN", "ghp_testtoken") + catalog = self._make_catalog(temp_dir) + req = catalog._make_request("https://internal.example.com/catalog.json") + assert "Authorization" not in req.headers + + def test_make_request_token_added_for_api_github_com(self, temp_dir, monkeypatch): + """GITHUB_TOKEN is attached for api.github.com URLs.""" + monkeypatch.setenv("GITHUB_TOKEN", "ghp_testtoken") + catalog = self._make_catalog(temp_dir) + req = catalog._make_request("https://api.github.com/repos/org/repo/releases/assets/1") + assert req.get_header("Authorization") == "token ghp_testtoken" + + def test_fetch_single_catalog_sends_auth_header(self, temp_dir, monkeypatch): + """_fetch_single_catalog passes Authorization header to urlopen for GitHub URLs.""" + from unittest.mock import patch, MagicMock + import io + + monkeypatch.setenv("GITHUB_TOKEN", "ghp_testtoken") + catalog = self._make_catalog(temp_dir) + + catalog_data = {"schema_version": "1.0", "extensions": {}} + mock_response = MagicMock() + mock_response.read.return_value = json.dumps(catalog_data).encode() + mock_response.__enter__ = lambda s: s + mock_response.__exit__ = MagicMock(return_value=False) + + captured = {} + + def fake_urlopen(req, timeout=None): + captured["req"] = req + return mock_response + + entry = CatalogEntry( + url="https://raw.githubusercontent.com/org/repo/main/catalog.json", + name="private", + priority=1, + install_allowed=True, + ) + + with patch("urllib.request.urlopen", fake_urlopen): + catalog._fetch_single_catalog(entry, force_refresh=True) + + assert "Authorization" in captured["req"].headers + assert captured["req"].headers["Authorization"] == "token ghp_testtoken" + + def test_download_extension_sends_auth_header(self, temp_dir, monkeypatch): + """download_extension passes Authorization header to urlopen for GitHub URLs.""" + from unittest.mock import patch, MagicMock + import zipfile, io + + monkeypatch.setenv("GITHUB_TOKEN", "ghp_testtoken") + catalog = self._make_catalog(temp_dir) + + # Build a minimal valid ZIP in memory + zip_buf = io.BytesIO() + with zipfile.ZipFile(zip_buf, "w") as zf: + zf.writestr("extension.yml", "id: test-ext\nname: Test\nversion: 1.0.0\n") + zip_bytes = zip_buf.getvalue() + + mock_response = MagicMock() + mock_response.read.return_value = zip_bytes + mock_response.__enter__ = lambda s: s + mock_response.__exit__ = MagicMock(return_value=False) + + captured = {} + + def fake_urlopen(req, timeout=None): + captured["req"] = req + return mock_response + + ext_info = { + "id": "test-ext", + "name": "Test Extension", + "version": "1.0.0", + "download_url": "https://github.com/org/repo/releases/download/v1/test-ext.zip", + } + + with patch.object(catalog, "get_extension_info", return_value=ext_info), \ + patch("urllib.request.urlopen", fake_urlopen): + catalog.download_extension("test-ext", target_dir=temp_dir) + + assert "Authorization" in captured["req"].headers + assert captured["req"].headers["Authorization"] == "token ghp_testtoken" + # ===== CatalogEntry Tests =====