From 0e4de13ed0b7abdad7a6bc5a1c7ebe12c6b421aa Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Wed, 11 Nov 2026 11:07:22 +0000 Subject: [PATCH 1/3] Check new documentation links against cached base reports Compare PR and merge-queue links with the exact base revision's Sphinx JSONL report. Escape and anchor known URLs so new paths and fragments remain checked, while preserving checks for local links and manual exclusions. Restore an exact-SHA baseline cache published only by main pushes. On a cache miss, fetch the base commit and generate its report in a temporary worktree rather than weakening enforcement. Accept existing broken links in that report, but reject missing or malformed reports. Make new-link failures block PRs and merge queues. Keep full checks on pushes informational and retain failing daily checks for site-wide link monitoring. Upload full reports and summarize failing URLs so maintainers can investigate outages without blocking unrelated changes. --- .github/workflows/test.yml | 60 ++++++++++++++- noxfile.py | 12 +++ scripts/linkcheck.py | 150 +++++++++++++++++++++++++++++++++++++ source/conf.py | 4 + 4 files changed, 223 insertions(+), 3 deletions(-) create mode 100644 scripts/linkcheck.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 75da24068d..989daffcd8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,7 +27,16 @@ jobs: timeout-minutes: 20 continue-on-error: >- ${{ fromJSON(matrix.continue-on-error) }} + env: + LINKCHECK_BASE_REF: >- + ${{ matrix.noxenv == 'linkcheck' && + (github.event.pull_request.base.sha || github.event.merge_group.base_sha) || '' }} + LINKCHECK_BASELINE: >- + ${{ matrix.noxenv == 'linkcheck' && + contains(fromJSON('["pull_request", "merge_group"]'), github.event_name) && + format('{0}/.nox/.cache/linkcheck-baseline.jsonl', github.workspace) || '' }} strategy: + fail-fast: false matrix: noxenv: - build @@ -35,10 +44,10 @@ jobs: - false include: - noxenv: linkcheck - # Don't block PRs, merge groups, or non-main pushes on unrelated linkcheck failures. + # Pushes monitor all links; only new links block PRs and merge groups. + # Scheduled runs remain blocking to report site-wide link failures. continue-on-error: >- - ${{ toJSON(contains(fromJSON('["merge_group", "pull_request"]'), github.event_name) || - (github.event_name == 'push' && github.ref_name != github.event.repository.default_branch)) }} + ${{ toJSON(github.event_name == 'push') }} steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 @@ -56,6 +65,13 @@ jobs: run: | python -m pip install --upgrade nox virtualenv + - name: Restore linkcheck baseline + if: env.LINKCHECK_BASE_REF != '' + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: .nox/.cache/linkcheck-baseline.jsonl + key: linkcheck-baseline-v1-${{ env.LINKCHECK_BASE_REF }} + - name: Nox ${{ matrix.noxenv }} env: # Authenticate github.com requests during linkcheck to avoid rate limits. @@ -63,6 +79,44 @@ jobs: run: | python -m nox -s ${{ matrix.noxenv }} + - name: Prepare report for baseline cache + id: baseline-report + if: >- + ${{ always() && matrix.noxenv == 'linkcheck' && github.event_name == 'push' && + github.ref_name == github.event.repository.default_branch }} + run: >- + python scripts/linkcheck.py publish --report build/output.json + --baseline .nox/.cache/linkcheck-baseline.jsonl + + - name: Save linkcheck baseline + if: ${{ always() && steps.baseline-report.outcome == 'success' }} + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: .nox/.cache/linkcheck-baseline.jsonl + key: linkcheck-baseline-v1-${{ github.sha }} + + - name: Upload full linkcheck reports + if: >- + ${{ always() && matrix.noxenv == 'linkcheck' && + (github.event_name == 'schedule' || (github.event_name == 'push' && + github.ref_name == github.event.repository.default_branch)) }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: linkcheck-reports + path: | + build/output.json + build/output.txt + if-no-files-found: warn + + - name: Summarize full linkcheck + if: >- + ${{ always() && matrix.noxenv == 'linkcheck' && + (github.event_name == 'schedule' || (github.event_name == 'push' && + github.ref_name == github.event.repository.default_branch)) }} + run: >- + python scripts/linkcheck.py summarize --report build/output.json + --summary "$GITHUB_STEP_SUMMARY" + check: # This job does nothing and is only used for the branch protection diff --git a/noxfile.py b/noxfile.py index 484a8d39a7..f7ce5166ba 100644 --- a/noxfile.py +++ b/noxfile.py @@ -4,6 +4,8 @@ # http://creativecommons.org/licenses/by-sa/3.0. +import os + import nox nox.options.sessions = [] @@ -77,6 +79,16 @@ def linkcheck(session): Check for broken links. """ session.install("-r", "requirements.txt") + if base := os.getenv("LINKCHECK_BASE_REF"): + session.run( + "python", + "scripts/linkcheck.py", + "prepare", + "--base", + base, + "--report", + os.environ["LINKCHECK_BASELINE"], + ) session.run( "sphinx-build", "-b", diff --git a/scripts/linkcheck.py b/scripts/linkcheck.py new file mode 100644 index 0000000000..89dec2a3f1 --- /dev/null +++ b/scripts/linkcheck.py @@ -0,0 +1,150 @@ +"""Prepare exact-URL linkcheck baselines and summarize monitoring reports.""" + +from __future__ import annotations + +import argparse +import html +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + + +def read_report(path: Path) -> list[dict]: + """Read Sphinx's JSONL report, rejecting unusable cache entries.""" + records = [] + for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + if not line.strip(): + continue + try: + record = json.loads(line) + except json.JSONDecodeError as exc: + raise ValueError(f"{path}:{number}: invalid JSON") from exc + if ( + not isinstance(record, dict) + or not isinstance(record.get("uri"), str) + or not record["uri"] + or not isinstance(record.get("status"), str) + ): + raise ValueError(f"{path}:{number}: invalid linkcheck record") + records.append(record) + return records + + +def baseline_patterns(path: Path) -> list[str]: + """Ignore known external URLs, without ignoring new suffixes or local links.""" + urls = { + record["uri"] + for record in read_report(path) + if record["uri"].startswith(("http://", "https://")) + } + print(f"Linkcheck baseline: {len(urls)} known external URLs from {path}") + return [r"\A" + re.escape(url) + r"\Z" for url in sorted(urls)] + + +def copy_report(source: Path, destination: Path) -> None: + read_report(source) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + + +def prepare_baseline(base: str, report: Path) -> None: + """Use the cached report or generate one at the exact base revision.""" + if not re.fullmatch(r"[0-9a-f]{40}", base): + raise ValueError("The linkcheck base must be a full Git commit SHA") + if report.exists(): + read_report(report) + print(f"Linkcheck baseline cache hit: {base}") + return + + print(f"Linkcheck baseline cache miss: generating report for {base}", flush=True) + if subprocess.run( + ["git", "cat-file", "-e", f"{base}^{{commit}}"], + check=False, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ).returncode: + subprocess.run( + ["git", "fetch", "--no-tags", "--depth=1", "origin", base], check=True + ) + + with tempfile.TemporaryDirectory(prefix="linkcheck-baseline-") as directory: + temporary = Path(directory) + checkout = temporary / "checkout" + output = temporary / "report" + subprocess.run( + ["git", "worktree", "add", "--detach", str(checkout), base], check=True + ) + try: + env = os.environ.copy() + env.pop("LINKCHECK_BASELINE", None) + env.pop("LINKCHECK_BASE_REF", None) + result = subprocess.run( + [ + sys.executable, + "-m", + "sphinx", + "-b", + "linkcheck", + "-d", + str(temporary / "doctrees"), + "-n", + "-W", + "--keep-going", + "source", + str(output), + ], + cwd=checkout, + env=env, + check=False, + ) + # Sphinx returns 1 for broken links or warnings. Those do not change + # which URLs existed at the base revision; a report is still required. + if result.returncode not in (0, 1): + result.check_returncode() + copy_report(output / "output.json", report) + finally: + subprocess.run( + ["git", "worktree", "remove", "--force", str(checkout)], check=True + ) + + +def summarize(report: Path, summary: Path) -> None: + records = read_report(report) + failures = [r for r in records if r["status"] in {"broken", "timeout"}] + lines = ["## Full linkcheck", "", f"{len(failures)} failing URLs.", ""] + for record in failures: + uri = html.escape(record["uri"]) + info = html.escape(str(record.get("info", record["status"]))).replace("\n", " ") + lines.append(f"- {uri}: {info}") + with summary.open("a", encoding="utf-8") as stream: + stream.write("\n".join(lines) + "\n") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + prepare = commands.add_parser("prepare") + prepare.add_argument("--base", required=True) + prepare.add_argument("--report", type=Path, required=True) + publish = commands.add_parser("publish") + publish.add_argument("--report", type=Path, required=True) + publish.add_argument("--baseline", type=Path, required=True) + summary = commands.add_parser("summarize") + summary.add_argument("--report", type=Path, required=True) + summary.add_argument("--summary", type=Path, required=True) + args = parser.parse_args() + if args.command == "prepare": + prepare_baseline(args.base, args.report.resolve()) + elif args.command == "publish": + copy_report(args.report, args.baseline) + else: + summarize(args.report, args.summary) + + +if __name__ == "__main__": + main() diff --git a/source/conf.py b/source/conf.py index e2ed462c8b..bcc76b1b1f 100644 --- a/source/conf.py +++ b/source/conf.py @@ -175,6 +175,10 @@ # Archived snapshots don't change, and the Wayback Machine rate-limits CI (429). r"https://web\.archive\.org/.*", ] +if _baseline := os.getenv("LINKCHECK_BASELINE"): + from scripts.linkcheck import baseline_patterns + + linkcheck_ignore.extend(baseline_patterns(pathlib.Path(_baseline))) linkcheck_retries = 2 linkcheck_timeout = 30 # Ignore anchors for common targets when we know they likely won't be found From 226450159849e8e39067c49558a6a9e36ece8a04 Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Wed, 11 Nov 2026 11:07:29 +0000 Subject: [PATCH 2/3] Test linkcheck filtering and baseline fallback with pytest Add a Nox test session to CI and use pytest fixtures and parametrization to cover exact URL matching, report validation, and monitoring summaries. Exercise Sphinx against a local HTTP server to prove existing broken URLs are skipped while new working and broken URLs are actually checked. Use temporary Git repositories to verify cache reuse, environment isolation, and worktree cleanup on success and failure. Include a shallow clone so fetching a missing base commit is tested as it occurs in CI. Keep network requests local and Git mutations confined to disposable repositories so contributors can run the suite against their own checkout. --- .github/workflows/test.yml | 1 + noxfile.py | 7 + tests/test_linkcheck.py | 255 +++++++++++++++++++++++++++++++++++++ 3 files changed, 263 insertions(+) create mode 100644 tests/test_linkcheck.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 989daffcd8..670ee66b95 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -40,6 +40,7 @@ jobs: matrix: noxenv: - build + - tests continue-on-error: - false include: diff --git a/noxfile.py b/noxfile.py index f7ce5166ba..3d34c41c20 100644 --- a/noxfile.py +++ b/noxfile.py @@ -105,6 +105,13 @@ def linkcheck(session): ) +@nox.session() +def tests(session): + """Test the linkcheck CI helpers and their Sphinx integration.""" + session.install("-r", "requirements.txt", "pytest>=8,<10") + session.run("python", "-m", "pytest", "tests", *session.posargs) + + @nox.session() def checkqa(session): """ diff --git a/tests/test_linkcheck.py b/tests/test_linkcheck.py new file mode 100644 index 0000000000..8ef2af4324 --- /dev/null +++ b/tests/test_linkcheck.py @@ -0,0 +1,255 @@ +import json +import re +import subprocess +import sys +from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from threading import Thread + +import pytest + +from scripts.linkcheck import ( + baseline_patterns, + prepare_baseline, + read_report, + summarize, +) + +ROOT = Path(__file__).resolve().parents[1] + + +def write_report(path, records): + path.write_text("".join(json.dumps(record) + "\n" for record in records)) + + +@pytest.mark.parametrize("status", ["working", "broken", "redirected", "ignored"]) +def test_exact_external_urls(tmp_path, status): + url = "https://example.org/docs_(v1)?a=1&b=2" + report = tmp_path / "output.json" + write_report( + report, + [ + {"uri": url, "status": status}, + {"uri": url, "status": status}, + {"uri": "/local/file", "status": "working"}, + {"uri": "mailto:docs@example.org", "status": "unchecked"}, + ], + ) + patterns = baseline_patterns(report) + assert len(patterns) == 1 + assert re.match(patterns[0], url) + for other in ( + url + "#missing-anchor", + url + "&new=value", + url + "/new-page", + url.replace("example.org", "exampleXorg"), + url + "\n", + "/local/file", + ): + assert not re.match(patterns[0], other) + + +@pytest.mark.parametrize( + "content", ["not JSON", "[]", "null", "{}", '{"uri": 1, "status": "broken"}'] +) +def test_invalid_reports(tmp_path, content): + report = tmp_path / "output.json" + report.write_text(content) + with pytest.raises(ValueError, match="output.json:1:"): + read_report(report) + + +def test_missing_and_empty_reports(tmp_path): + report = tmp_path / "output.json" + with pytest.raises(FileNotFoundError): + read_report(report) + report.write_text("\n") + assert baseline_patterns(report) == [] + + +def test_summary(tmp_path): + report, summary = tmp_path / "output.json", tmp_path / "summary.md" + write_report( + report, + [ + {"uri": "https://example.org/ok", "status": "working"}, + {"uri": "https://example.org/", "status": "broken", "info": "404"}, + {"uri": "https://example.org/slow", "status": "timeout"}, + ], + ) + summary.write_text("Existing summary\n") + summarize(report, summary) + text = summary.read_text() + assert text.startswith("Existing summary\n") + assert "2 failing URLs" in text + assert "<bad>" in text + assert "https://example.org/slow" in text + assert "https://example.org/ok" not in text + + +@pytest.fixture +def server(): + requests = [] + + class Handler(BaseHTTPRequestHandler): + def do_HEAD(self): + requests.append(self.path) + self.send_response(200 if self.path.startswith("/ok") else 404) + self.end_headers() + + def log_message(self, *args): + pass + + with ThreadingHTTPServer(("127.0.0.1", 0), Handler) as http: + thread = Thread(target=http.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{http.server_port}", requests + finally: + http.shutdown() + thread.join() + + +def documentation(directory, text, *, base=False): + source = directory / "source" + source.mkdir() + configuration = ( + "import os, sys\n" + f"sys.path.insert(0, {str(ROOT)!r})\n" + "from pathlib import Path\n" + "from scripts.linkcheck import baseline_patterns\n" + "linkcheck_ignore = []\n" + "if baseline := os.getenv('LINKCHECK_BASELINE'):\n" + " linkcheck_ignore = baseline_patterns(Path(baseline))\n" + "linkcheck_retries = 1\n" + "linkcheck_timeout = 2\n" + ) + if base: + configuration += ( + "assert not os.getenv('LINKCHECK_BASELINE')\n" + "assert not os.getenv('LINKCHECK_BASE_REF')\n" + ) + (source / "conf.py").write_text(configuration) + (source / "index.rst").write_text("Links\n=====\n\n" + text) + return source + + +def sphinx(source, output): + return subprocess.run( + [ + sys.executable, + "-m", + "sphinx", + "-q", + "-b", + "linkcheck", + "-d", + str(output / "doctrees"), + str(source), + str(output), + ], + capture_output=True, + text=True, + check=False, + ) + + +def test_sphinx_checks_only_new_urls(tmp_path, server, monkeypatch): + url, requests = server + monkeypatch.delenv("LINKCHECK_BASELINE", raising=False) + source = documentation(tmp_path, f"{url}/old-broken\n\n{url}/ok\n") + full = tmp_path / "full" + assert sphinx(source, full).returncode == 1 + assert "/old-broken" in requests + + monkeypatch.setenv("LINKCHECK_BASELINE", str(full / "output.json")) + requests.clear() + with (source / "index.rst").open("a") as stream: + stream.write(f"\n{url}/ok?new\n") + result = sphinx(source, tmp_path / "filtered") + assert result.returncode == 0, result.stdout + result.stderr + assert requests == ["/ok?new"] + + requests.clear() + with (source / "index.rst").open("a") as stream: + stream.write(f"\n{url}/new-broken\n") + assert sphinx(source, tmp_path / "new-broken").returncode == 1 + assert "/new-broken" in requests + assert "/old-broken" not in requests + + +def git(*args): + return subprocess.check_output(["git", *args], text=True).strip() + + +@contextmanager +def repository(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + git("init", "-q") + git("add", "source") + git( + "-c", + "user.name=Test", + "-c", + "user.email=test@example.org", + "commit", + "-qm", + "Docs", + ) + yield git("rev-parse", "HEAD") + assert git("worktree", "list", "--porcelain").count("worktree ") == 1 + + +def test_generate_baseline_and_reuse_cache(tmp_path, server, monkeypatch): + url, requests = server + documentation(tmp_path, f"{url}/old-broken\n", base=True) + report = tmp_path / "cache" / "baseline.jsonl" + with repository(tmp_path, monkeypatch) as base: + monkeypatch.setenv("LINKCHECK_BASELINE", "must-not-be-read") + monkeypatch.setenv("LINKCHECK_BASE_REF", "must-not-be-read") + prepare_baseline(base, report) + assert read_report(report)[0]["uri"] == f"{url}/old-broken" + assert "/old-broken" in requests + requests.clear() + prepare_baseline(base, report) + assert not requests + + +def test_baseline_failure_cleans_up_worktree(tmp_path, monkeypatch): + source = documentation(tmp_path, "No links.\n") + (source / "conf.py").write_text("raise RuntimeError('broken configuration')\n") + with repository(tmp_path, monkeypatch) as base: + report = tmp_path / "baseline.jsonl" + with pytest.raises(subprocess.CalledProcessError): + prepare_baseline(base, report) + assert not report.exists() + + +def test_fetch_baseline_in_shallow_checkout(tmp_path, server, monkeypatch): + url, _ = server + original = tmp_path / "original" + original.mkdir() + source = documentation(original, f"{url}/old-broken\n", base=True) + with repository(original, monkeypatch) as base: + (source / "index.rst").write_text("Current\n=======\n\nNo links.\n") + git("add", "source") + git( + "-c", + "user.name=Test", + "-c", + "user.email=test@example.org", + "commit", + "-qm", + "Change docs", + ) + checkout = tmp_path / "shallow" + git("clone", "--depth=1", original.as_uri(), str(checkout)) + with monkeypatch.context() as context: + context.chdir(checkout) + with pytest.raises(subprocess.CalledProcessError): + git("cat-file", "-e", base) + report = tmp_path / "baseline.jsonl" + prepare_baseline(base, report) + assert read_report(report)[0]["uri"] == f"{url}/old-broken" + assert git("worktree", "list", "--porcelain").count("worktree ") == 1 From 17794c5019582e71ccb4da5b54c508cc5259ea50 Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Wed, 11 Nov 2026 11:07:38 +0000 Subject: [PATCH 3/3] Document linkcheck policy and contributor test commands Explain how to run full local linkcheck and the pytest suite through Nox, including report locations and the use of temporary repositories in tests. Describe exact-base filtering and cache-miss generation for PRs and merge queues. Distinguish informational full checks after main pushes from failing daily monitoring, and point contributors to uploaded reports and job summaries when investigating link failures. --- source/contribute.rst | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/source/contribute.rst b/source/contribute.rst index 112643d98c..a4560f156d 100644 --- a/source/contribute.rst +++ b/source/contribute.rst @@ -158,6 +158,41 @@ command: The guide will be browsable via http://localhost:8000. +Checking links +============== + +To check links locally, run: + +.. code-block:: bash + + nox -s linkcheck + +This checks all links, except those explicitly excluded in ``source/conf.py``. +Reports are written to ``build/output.txt`` and ``build/output.json``. + +In CI, pull requests and merge queues check external URLs that are absent from +the base revision's linkcheck report. New paths and fragments count as new URLs. +The base report is cached, or generated on demand if the cache is missing. +Local links are still checked. + +After pushes to ``main``, full linkcheck reports failures without failing the +aggregate ``check`` job. The daily run at 06:00 UTC checks all links and fails on +broken links. Both runs upload reports and list failing URLs in the job summary. + + +Testing the linkcheck tooling +============================= + +When changing the linkcheck tooling, run its pytest suite: + +.. code-block:: bash + + nox -s tests + +Nox installs the test dependencies automatically. The tests use a local HTTP +server and temporary Git repositories. They also run in CI. + + Where the guide is deployed ===========================