diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 75da24068d..670ee66b95 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -27,18 +27,28 @@ jobs:
timeout-minutes: 20
continue-on-error: >-
${{ fromJSON(matrix.continue-on-error) }}
+ env:
+ LINKCHECK_BASE_REF: >-
+ ${{ matrix.noxenv == 'linkcheck' &&
+ (github.event.pull_request.base.sha || github.event.merge_group.base_sha) || '' }}
+ LINKCHECK_BASELINE: >-
+ ${{ matrix.noxenv == 'linkcheck' &&
+ contains(fromJSON('["pull_request", "merge_group"]'), github.event_name) &&
+ format('{0}/.nox/.cache/linkcheck-baseline.jsonl', github.workspace) || '' }}
strategy:
+ fail-fast: false
matrix:
noxenv:
- build
+ - tests
continue-on-error:
- false
include:
- noxenv: linkcheck
- # Don't block PRs, merge groups, or non-main pushes on unrelated linkcheck failures.
+ # Pushes monitor all links; only new links block PRs and merge groups.
+ # Scheduled runs remain blocking to report site-wide link failures.
continue-on-error: >-
- ${{ toJSON(contains(fromJSON('["merge_group", "pull_request"]'), github.event_name) ||
- (github.event_name == 'push' && github.ref_name != github.event.repository.default_branch)) }}
+ ${{ toJSON(github.event_name == 'push') }}
steps:
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
@@ -56,6 +66,13 @@ jobs:
run: |
python -m pip install --upgrade nox virtualenv
+ - name: Restore linkcheck baseline
+ if: env.LINKCHECK_BASE_REF != ''
+ uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+ with:
+ path: .nox/.cache/linkcheck-baseline.jsonl
+ key: linkcheck-baseline-v1-${{ env.LINKCHECK_BASE_REF }}
+
- name: Nox ${{ matrix.noxenv }}
env:
# Authenticate github.com requests during linkcheck to avoid rate limits.
@@ -63,6 +80,44 @@ jobs:
run: |
python -m nox -s ${{ matrix.noxenv }}
+ - name: Prepare report for baseline cache
+ id: baseline-report
+ if: >-
+ ${{ always() && matrix.noxenv == 'linkcheck' && github.event_name == 'push' &&
+ github.ref_name == github.event.repository.default_branch }}
+ run: >-
+ python scripts/linkcheck.py publish --report build/output.json
+ --baseline .nox/.cache/linkcheck-baseline.jsonl
+
+ - name: Save linkcheck baseline
+ if: ${{ always() && steps.baseline-report.outcome == 'success' }}
+ uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+ with:
+ path: .nox/.cache/linkcheck-baseline.jsonl
+ key: linkcheck-baseline-v1-${{ github.sha }}
+
+ - name: Upload full linkcheck reports
+ if: >-
+ ${{ always() && matrix.noxenv == 'linkcheck' &&
+ (github.event_name == 'schedule' || (github.event_name == 'push' &&
+ github.ref_name == github.event.repository.default_branch)) }}
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+ with:
+ name: linkcheck-reports
+ path: |
+ build/output.json
+ build/output.txt
+ if-no-files-found: warn
+
+ - name: Summarize full linkcheck
+ if: >-
+ ${{ always() && matrix.noxenv == 'linkcheck' &&
+ (github.event_name == 'schedule' || (github.event_name == 'push' &&
+ github.ref_name == github.event.repository.default_branch)) }}
+ run: >-
+ python scripts/linkcheck.py summarize --report build/output.json
+ --summary "$GITHUB_STEP_SUMMARY"
+
check:
# This job does nothing and is only used for the branch protection
diff --git a/noxfile.py b/noxfile.py
index 484a8d39a7..3d34c41c20 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -4,6 +4,8 @@
# http://creativecommons.org/licenses/by-sa/3.0.
+import os
+
import nox
nox.options.sessions = []
@@ -77,6 +79,16 @@ def linkcheck(session):
Check for broken links.
"""
session.install("-r", "requirements.txt")
+ if base := os.getenv("LINKCHECK_BASE_REF"):
+ session.run(
+ "python",
+ "scripts/linkcheck.py",
+ "prepare",
+ "--base",
+ base,
+ "--report",
+ os.environ["LINKCHECK_BASELINE"],
+ )
session.run(
"sphinx-build",
"-b",
@@ -93,6 +105,13 @@ def linkcheck(session):
)
+@nox.session()
+def tests(session):
+ """Test the linkcheck CI helpers and their Sphinx integration."""
+ session.install("-r", "requirements.txt", "pytest>=8,<10")
+ session.run("python", "-m", "pytest", "tests", *session.posargs)
+
+
@nox.session()
def checkqa(session):
"""
diff --git a/scripts/linkcheck.py b/scripts/linkcheck.py
new file mode 100644
index 0000000000..89dec2a3f1
--- /dev/null
+++ b/scripts/linkcheck.py
@@ -0,0 +1,150 @@
+"""Prepare exact-URL linkcheck baselines and summarize monitoring reports."""
+
+from __future__ import annotations
+
+import argparse
+import html
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+
+def read_report(path: Path) -> list[dict]:
+ """Read Sphinx's JSONL report, rejecting unusable cache entries."""
+ records = []
+ for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
+ if not line.strip():
+ continue
+ try:
+ record = json.loads(line)
+ except json.JSONDecodeError as exc:
+ raise ValueError(f"{path}:{number}: invalid JSON") from exc
+ if (
+ not isinstance(record, dict)
+ or not isinstance(record.get("uri"), str)
+ or not record["uri"]
+ or not isinstance(record.get("status"), str)
+ ):
+ raise ValueError(f"{path}:{number}: invalid linkcheck record")
+ records.append(record)
+ return records
+
+
+def baseline_patterns(path: Path) -> list[str]:
+ """Ignore known external URLs, without ignoring new suffixes or local links."""
+ urls = {
+ record["uri"]
+ for record in read_report(path)
+ if record["uri"].startswith(("http://", "https://"))
+ }
+ print(f"Linkcheck baseline: {len(urls)} known external URLs from {path}")
+ return [r"\A" + re.escape(url) + r"\Z" for url in sorted(urls)]
+
+
+def copy_report(source: Path, destination: Path) -> None:
+ read_report(source)
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ shutil.copyfile(source, destination)
+
+
+def prepare_baseline(base: str, report: Path) -> None:
+ """Use the cached report or generate one at the exact base revision."""
+ if not re.fullmatch(r"[0-9a-f]{40}", base):
+ raise ValueError("The linkcheck base must be a full Git commit SHA")
+ if report.exists():
+ read_report(report)
+ print(f"Linkcheck baseline cache hit: {base}")
+ return
+
+ print(f"Linkcheck baseline cache miss: generating report for {base}", flush=True)
+ if subprocess.run(
+ ["git", "cat-file", "-e", f"{base}^{{commit}}"],
+ check=False,
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ ).returncode:
+ subprocess.run(
+ ["git", "fetch", "--no-tags", "--depth=1", "origin", base], check=True
+ )
+
+ with tempfile.TemporaryDirectory(prefix="linkcheck-baseline-") as directory:
+ temporary = Path(directory)
+ checkout = temporary / "checkout"
+ output = temporary / "report"
+ subprocess.run(
+ ["git", "worktree", "add", "--detach", str(checkout), base], check=True
+ )
+ try:
+ env = os.environ.copy()
+ env.pop("LINKCHECK_BASELINE", None)
+ env.pop("LINKCHECK_BASE_REF", None)
+ result = subprocess.run(
+ [
+ sys.executable,
+ "-m",
+ "sphinx",
+ "-b",
+ "linkcheck",
+ "-d",
+ str(temporary / "doctrees"),
+ "-n",
+ "-W",
+ "--keep-going",
+ "source",
+ str(output),
+ ],
+ cwd=checkout,
+ env=env,
+ check=False,
+ )
+ # Sphinx returns 1 for broken links or warnings. Those do not change
+ # which URLs existed at the base revision; a report is still required.
+ if result.returncode not in (0, 1):
+ result.check_returncode()
+ copy_report(output / "output.json", report)
+ finally:
+ subprocess.run(
+ ["git", "worktree", "remove", "--force", str(checkout)], check=True
+ )
+
+
+def summarize(report: Path, summary: Path) -> None:
+ records = read_report(report)
+ failures = [r for r in records if r["status"] in {"broken", "timeout"}]
+ lines = ["## Full linkcheck", "", f"{len(failures)} failing URLs.", ""]
+ for record in failures:
+ uri = html.escape(record["uri"])
+ info = html.escape(str(record.get("info", record["status"]))).replace("\n", " ")
+ lines.append(f"- {uri}: {info}")
+ with summary.open("a", encoding="utf-8") as stream:
+ stream.write("\n".join(lines) + "\n")
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description=__doc__)
+ commands = parser.add_subparsers(dest="command", required=True)
+ prepare = commands.add_parser("prepare")
+ prepare.add_argument("--base", required=True)
+ prepare.add_argument("--report", type=Path, required=True)
+ publish = commands.add_parser("publish")
+ publish.add_argument("--report", type=Path, required=True)
+ publish.add_argument("--baseline", type=Path, required=True)
+ summary = commands.add_parser("summarize")
+ summary.add_argument("--report", type=Path, required=True)
+ summary.add_argument("--summary", type=Path, required=True)
+ args = parser.parse_args()
+ if args.command == "prepare":
+ prepare_baseline(args.base, args.report.resolve())
+ elif args.command == "publish":
+ copy_report(args.report, args.baseline)
+ else:
+ summarize(args.report, args.summary)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/source/conf.py b/source/conf.py
index e2ed462c8b..bcc76b1b1f 100644
--- a/source/conf.py
+++ b/source/conf.py
@@ -175,6 +175,10 @@
# Archived snapshots don't change, and the Wayback Machine rate-limits CI (429).
r"https://web\.archive\.org/.*",
]
+if _baseline := os.getenv("LINKCHECK_BASELINE"):
+ from scripts.linkcheck import baseline_patterns
+
+ linkcheck_ignore.extend(baseline_patterns(pathlib.Path(_baseline)))
linkcheck_retries = 2
linkcheck_timeout = 30
# Ignore anchors for common targets when we know they likely won't be found
diff --git a/source/contribute.rst b/source/contribute.rst
index 112643d98c..a4560f156d 100644
--- a/source/contribute.rst
+++ b/source/contribute.rst
@@ -158,6 +158,41 @@ command:
The guide will be browsable via http://localhost:8000.
+Checking links
+==============
+
+To check links locally, run:
+
+.. code-block:: bash
+
+ nox -s linkcheck
+
+This checks all links, except those explicitly excluded in ``source/conf.py``.
+Reports are written to ``build/output.txt`` and ``build/output.json``.
+
+In CI, pull requests and merge queues check external URLs that are absent from
+the base revision's linkcheck report. New paths and fragments count as new URLs.
+The base report is cached, or generated on demand if the cache is missing.
+Local links are still checked.
+
+After pushes to ``main``, full linkcheck reports failures without failing the
+aggregate ``check`` job. The daily run at 06:00 UTC checks all links and fails on
+broken links. Both runs upload reports and list failing URLs in the job summary.
+
+
+Testing the linkcheck tooling
+=============================
+
+When changing the linkcheck tooling, run its pytest suite:
+
+.. code-block:: bash
+
+ nox -s tests
+
+Nox installs the test dependencies automatically. The tests use a local HTTP
+server and temporary Git repositories. They also run in CI.
+
+
Where the guide is deployed
===========================
diff --git a/tests/test_linkcheck.py b/tests/test_linkcheck.py
new file mode 100644
index 0000000000..8ef2af4324
--- /dev/null
+++ b/tests/test_linkcheck.py
@@ -0,0 +1,255 @@
+import json
+import re
+import subprocess
+import sys
+from contextlib import contextmanager
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+from threading import Thread
+
+import pytest
+
+from scripts.linkcheck import (
+ baseline_patterns,
+ prepare_baseline,
+ read_report,
+ summarize,
+)
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def write_report(path, records):
+ path.write_text("".join(json.dumps(record) + "\n" for record in records))
+
+
+@pytest.mark.parametrize("status", ["working", "broken", "redirected", "ignored"])
+def test_exact_external_urls(tmp_path, status):
+ url = "https://example.org/docs_(v1)?a=1&b=2"
+ report = tmp_path / "output.json"
+ write_report(
+ report,
+ [
+ {"uri": url, "status": status},
+ {"uri": url, "status": status},
+ {"uri": "/local/file", "status": "working"},
+ {"uri": "mailto:docs@example.org", "status": "unchecked"},
+ ],
+ )
+ patterns = baseline_patterns(report)
+ assert len(patterns) == 1
+ assert re.match(patterns[0], url)
+ for other in (
+ url + "#missing-anchor",
+ url + "&new=value",
+ url + "/new-page",
+ url.replace("example.org", "exampleXorg"),
+ url + "\n",
+ "/local/file",
+ ):
+ assert not re.match(patterns[0], other)
+
+
+@pytest.mark.parametrize(
+ "content", ["not JSON", "[]", "null", "{}", '{"uri": 1, "status": "broken"}']
+)
+def test_invalid_reports(tmp_path, content):
+ report = tmp_path / "output.json"
+ report.write_text(content)
+ with pytest.raises(ValueError, match="output.json:1:"):
+ read_report(report)
+
+
+def test_missing_and_empty_reports(tmp_path):
+ report = tmp_path / "output.json"
+ with pytest.raises(FileNotFoundError):
+ read_report(report)
+ report.write_text("\n")
+ assert baseline_patterns(report) == []
+
+
+def test_summary(tmp_path):
+ report, summary = tmp_path / "output.json", tmp_path / "summary.md"
+ write_report(
+ report,
+ [
+ {"uri": "https://example.org/ok", "status": "working"},
+ {"uri": "https://example.org/", "status": "broken", "info": "404"},
+ {"uri": "https://example.org/slow", "status": "timeout"},
+ ],
+ )
+ summary.write_text("Existing summary\n")
+ summarize(report, summary)
+ text = summary.read_text()
+ assert text.startswith("Existing summary\n")
+ assert "2 failing URLs" in text
+ assert "<bad>" in text
+ assert "https://example.org/slow" in text
+ assert "https://example.org/ok" not in text
+
+
+@pytest.fixture
+def server():
+ requests = []
+
+ class Handler(BaseHTTPRequestHandler):
+ def do_HEAD(self):
+ requests.append(self.path)
+ self.send_response(200 if self.path.startswith("/ok") else 404)
+ self.end_headers()
+
+ def log_message(self, *args):
+ pass
+
+ with ThreadingHTTPServer(("127.0.0.1", 0), Handler) as http:
+ thread = Thread(target=http.serve_forever, daemon=True)
+ thread.start()
+ try:
+ yield f"http://127.0.0.1:{http.server_port}", requests
+ finally:
+ http.shutdown()
+ thread.join()
+
+
+def documentation(directory, text, *, base=False):
+ source = directory / "source"
+ source.mkdir()
+ configuration = (
+ "import os, sys\n"
+ f"sys.path.insert(0, {str(ROOT)!r})\n"
+ "from pathlib import Path\n"
+ "from scripts.linkcheck import baseline_patterns\n"
+ "linkcheck_ignore = []\n"
+ "if baseline := os.getenv('LINKCHECK_BASELINE'):\n"
+ " linkcheck_ignore = baseline_patterns(Path(baseline))\n"
+ "linkcheck_retries = 1\n"
+ "linkcheck_timeout = 2\n"
+ )
+ if base:
+ configuration += (
+ "assert not os.getenv('LINKCHECK_BASELINE')\n"
+ "assert not os.getenv('LINKCHECK_BASE_REF')\n"
+ )
+ (source / "conf.py").write_text(configuration)
+ (source / "index.rst").write_text("Links\n=====\n\n" + text)
+ return source
+
+
+def sphinx(source, output):
+ return subprocess.run(
+ [
+ sys.executable,
+ "-m",
+ "sphinx",
+ "-q",
+ "-b",
+ "linkcheck",
+ "-d",
+ str(output / "doctrees"),
+ str(source),
+ str(output),
+ ],
+ capture_output=True,
+ text=True,
+ check=False,
+ )
+
+
+def test_sphinx_checks_only_new_urls(tmp_path, server, monkeypatch):
+ url, requests = server
+ monkeypatch.delenv("LINKCHECK_BASELINE", raising=False)
+ source = documentation(tmp_path, f"{url}/old-broken\n\n{url}/ok\n")
+ full = tmp_path / "full"
+ assert sphinx(source, full).returncode == 1
+ assert "/old-broken" in requests
+
+ monkeypatch.setenv("LINKCHECK_BASELINE", str(full / "output.json"))
+ requests.clear()
+ with (source / "index.rst").open("a") as stream:
+ stream.write(f"\n{url}/ok?new\n")
+ result = sphinx(source, tmp_path / "filtered")
+ assert result.returncode == 0, result.stdout + result.stderr
+ assert requests == ["/ok?new"]
+
+ requests.clear()
+ with (source / "index.rst").open("a") as stream:
+ stream.write(f"\n{url}/new-broken\n")
+ assert sphinx(source, tmp_path / "new-broken").returncode == 1
+ assert "/new-broken" in requests
+ assert "/old-broken" not in requests
+
+
+def git(*args):
+ return subprocess.check_output(["git", *args], text=True).strip()
+
+
+@contextmanager
+def repository(tmp_path, monkeypatch):
+ monkeypatch.chdir(tmp_path)
+ git("init", "-q")
+ git("add", "source")
+ git(
+ "-c",
+ "user.name=Test",
+ "-c",
+ "user.email=test@example.org",
+ "commit",
+ "-qm",
+ "Docs",
+ )
+ yield git("rev-parse", "HEAD")
+ assert git("worktree", "list", "--porcelain").count("worktree ") == 1
+
+
+def test_generate_baseline_and_reuse_cache(tmp_path, server, monkeypatch):
+ url, requests = server
+ documentation(tmp_path, f"{url}/old-broken\n", base=True)
+ report = tmp_path / "cache" / "baseline.jsonl"
+ with repository(tmp_path, monkeypatch) as base:
+ monkeypatch.setenv("LINKCHECK_BASELINE", "must-not-be-read")
+ monkeypatch.setenv("LINKCHECK_BASE_REF", "must-not-be-read")
+ prepare_baseline(base, report)
+ assert read_report(report)[0]["uri"] == f"{url}/old-broken"
+ assert "/old-broken" in requests
+ requests.clear()
+ prepare_baseline(base, report)
+ assert not requests
+
+
+def test_baseline_failure_cleans_up_worktree(tmp_path, monkeypatch):
+ source = documentation(tmp_path, "No links.\n")
+ (source / "conf.py").write_text("raise RuntimeError('broken configuration')\n")
+ with repository(tmp_path, monkeypatch) as base:
+ report = tmp_path / "baseline.jsonl"
+ with pytest.raises(subprocess.CalledProcessError):
+ prepare_baseline(base, report)
+ assert not report.exists()
+
+
+def test_fetch_baseline_in_shallow_checkout(tmp_path, server, monkeypatch):
+ url, _ = server
+ original = tmp_path / "original"
+ original.mkdir()
+ source = documentation(original, f"{url}/old-broken\n", base=True)
+ with repository(original, monkeypatch) as base:
+ (source / "index.rst").write_text("Current\n=======\n\nNo links.\n")
+ git("add", "source")
+ git(
+ "-c",
+ "user.name=Test",
+ "-c",
+ "user.email=test@example.org",
+ "commit",
+ "-qm",
+ "Change docs",
+ )
+ checkout = tmp_path / "shallow"
+ git("clone", "--depth=1", original.as_uri(), str(checkout))
+ with monkeypatch.context() as context:
+ context.chdir(checkout)
+ with pytest.raises(subprocess.CalledProcessError):
+ git("cat-file", "-e", base)
+ report = tmp_path / "baseline.jsonl"
+ prepare_baseline(base, report)
+ assert read_report(report)[0]["uri"] == f"{url}/old-broken"
+ assert git("worktree", "list", "--porcelain").count("worktree ") == 1