diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 75da24068d..670ee66b95 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,18 +27,28 @@ jobs: timeout-minutes: 20 continue-on-error: >- ${{ fromJSON(matrix.continue-on-error) }} + env: + LINKCHECK_BASE_REF: >- + ${{ matrix.noxenv == 'linkcheck' && + (github.event.pull_request.base.sha || github.event.merge_group.base_sha) || '' }} + LINKCHECK_BASELINE: >- + ${{ matrix.noxenv == 'linkcheck' && + contains(fromJSON('["pull_request", "merge_group"]'), github.event_name) && + format('{0}/.nox/.cache/linkcheck-baseline.jsonl', github.workspace) || '' }} strategy: + fail-fast: false matrix: noxenv: - build + - tests continue-on-error: - false include: - noxenv: linkcheck - # Don't block PRs, merge groups, or non-main pushes on unrelated linkcheck failures. + # Pushes monitor all links; only new links block PRs and merge groups. + # Scheduled runs remain blocking to report site-wide link failures. continue-on-error: >- - ${{ toJSON(contains(fromJSON('["merge_group", "pull_request"]'), github.event_name) || - (github.event_name == 'push' && github.ref_name != github.event.repository.default_branch)) }} + ${{ toJSON(github.event_name == 'push') }} steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 @@ -56,6 +66,13 @@ jobs: run: | python -m pip install --upgrade nox virtualenv + - name: Restore linkcheck baseline + if: env.LINKCHECK_BASE_REF != '' + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: .nox/.cache/linkcheck-baseline.jsonl + key: linkcheck-baseline-v1-${{ env.LINKCHECK_BASE_REF }} + - name: Nox ${{ matrix.noxenv }} env: # Authenticate github.com requests during linkcheck to avoid rate limits. @@ -63,6 +80,44 @@ jobs: run: | python -m nox -s ${{ matrix.noxenv }} + - name: Prepare report for baseline cache + id: baseline-report + if: >- + ${{ always() && matrix.noxenv == 'linkcheck' && github.event_name == 'push' && + github.ref_name == github.event.repository.default_branch }} + run: >- + python scripts/linkcheck.py publish --report build/output.json + --baseline .nox/.cache/linkcheck-baseline.jsonl + + - name: Save linkcheck baseline + if: ${{ always() && steps.baseline-report.outcome == 'success' }} + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: .nox/.cache/linkcheck-baseline.jsonl + key: linkcheck-baseline-v1-${{ github.sha }} + + - name: Upload full linkcheck reports + if: >- + ${{ always() && matrix.noxenv == 'linkcheck' && + (github.event_name == 'schedule' || (github.event_name == 'push' && + github.ref_name == github.event.repository.default_branch)) }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: linkcheck-reports + path: | + build/output.json + build/output.txt + if-no-files-found: warn + + - name: Summarize full linkcheck + if: >- + ${{ always() && matrix.noxenv == 'linkcheck' && + (github.event_name == 'schedule' || (github.event_name == 'push' && + github.ref_name == github.event.repository.default_branch)) }} + run: >- + python scripts/linkcheck.py summarize --report build/output.json + --summary "$GITHUB_STEP_SUMMARY" + check: # This job does nothing and is only used for the branch protection diff --git a/noxfile.py b/noxfile.py index 484a8d39a7..3d34c41c20 100644 --- a/noxfile.py +++ b/noxfile.py @@ -4,6 +4,8 @@ # http://creativecommons.org/licenses/by-sa/3.0. +import os + import nox nox.options.sessions = [] @@ -77,6 +79,16 @@ def linkcheck(session): Check for broken links. """ session.install("-r", "requirements.txt") + if base := os.getenv("LINKCHECK_BASE_REF"): + session.run( + "python", + "scripts/linkcheck.py", + "prepare", + "--base", + base, + "--report", + os.environ["LINKCHECK_BASELINE"], + ) session.run( "sphinx-build", "-b", @@ -93,6 +105,13 @@ def linkcheck(session): ) +@nox.session() +def tests(session): + """Test the linkcheck CI helpers and their Sphinx integration.""" + session.install("-r", "requirements.txt", "pytest>=8,<10") + session.run("python", "-m", "pytest", "tests", *session.posargs) + + @nox.session() def checkqa(session): """ diff --git a/scripts/linkcheck.py b/scripts/linkcheck.py new file mode 100644 index 0000000000..89dec2a3f1 --- /dev/null +++ b/scripts/linkcheck.py @@ -0,0 +1,150 @@ +"""Prepare exact-URL linkcheck baselines and summarize monitoring reports.""" + +from __future__ import annotations + +import argparse +import html +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + + +def read_report(path: Path) -> list[dict]: + """Read Sphinx's JSONL report, rejecting unusable cache entries.""" + records = [] + for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + if not line.strip(): + continue + try: + record = json.loads(line) + except json.JSONDecodeError as exc: + raise ValueError(f"{path}:{number}: invalid JSON") from exc + if ( + not isinstance(record, dict) + or not isinstance(record.get("uri"), str) + or not record["uri"] + or not isinstance(record.get("status"), str) + ): + raise ValueError(f"{path}:{number}: invalid linkcheck record") + records.append(record) + return records + + +def baseline_patterns(path: Path) -> list[str]: + """Ignore known external URLs, without ignoring new suffixes or local links.""" + urls = { + record["uri"] + for record in read_report(path) + if record["uri"].startswith(("http://", "https://")) + } + print(f"Linkcheck baseline: {len(urls)} known external URLs from {path}") + return [r"\A" + re.escape(url) + r"\Z" for url in sorted(urls)] + + +def copy_report(source: Path, destination: Path) -> None: + read_report(source) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + + +def prepare_baseline(base: str, report: Path) -> None: + """Use the cached report or generate one at the exact base revision.""" + if not re.fullmatch(r"[0-9a-f]{40}", base): + raise ValueError("The linkcheck base must be a full Git commit SHA") + if report.exists(): + read_report(report) + print(f"Linkcheck baseline cache hit: {base}") + return + + print(f"Linkcheck baseline cache miss: generating report for {base}", flush=True) + if subprocess.run( + ["git", "cat-file", "-e", f"{base}^{{commit}}"], + check=False, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ).returncode: + subprocess.run( + ["git", "fetch", "--no-tags", "--depth=1", "origin", base], check=True + ) + + with tempfile.TemporaryDirectory(prefix="linkcheck-baseline-") as directory: + temporary = Path(directory) + checkout = temporary / "checkout" + output = temporary / "report" + subprocess.run( + ["git", "worktree", "add", "--detach", str(checkout), base], check=True + ) + try: + env = os.environ.copy() + env.pop("LINKCHECK_BASELINE", None) + env.pop("LINKCHECK_BASE_REF", None) + result = subprocess.run( + [ + sys.executable, + "-m", + "sphinx", + "-b", + "linkcheck", + "-d", + str(temporary / "doctrees"), + "-n", + "-W", + "--keep-going", + "source", + str(output), + ], + cwd=checkout, + env=env, + check=False, + ) + # Sphinx returns 1 for broken links or warnings. Those do not change + # which URLs existed at the base revision; a report is still required. + if result.returncode not in (0, 1): + result.check_returncode() + copy_report(output / "output.json", report) + finally: + subprocess.run( + ["git", "worktree", "remove", "--force", str(checkout)], check=True + ) + + +def summarize(report: Path, summary: Path) -> None: + records = read_report(report) + failures = [r for r in records if r["status"] in {"broken", "timeout"}] + lines = ["## Full linkcheck", "", f"{len(failures)} failing URLs.", ""] + for record in failures: + uri = html.escape(record["uri"]) + info = html.escape(str(record.get("info", record["status"]))).replace("\n", " ") + lines.append(f"- {uri}: {info}") + with summary.open("a", encoding="utf-8") as stream: + stream.write("\n".join(lines) + "\n") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + prepare = commands.add_parser("prepare") + prepare.add_argument("--base", required=True) + prepare.add_argument("--report", type=Path, required=True) + publish = commands.add_parser("publish") + publish.add_argument("--report", type=Path, required=True) + publish.add_argument("--baseline", type=Path, required=True) + summary = commands.add_parser("summarize") + summary.add_argument("--report", type=Path, required=True) + summary.add_argument("--summary", type=Path, required=True) + args = parser.parse_args() + if args.command == "prepare": + prepare_baseline(args.base, args.report.resolve()) + elif args.command == "publish": + copy_report(args.report, args.baseline) + else: + summarize(args.report, args.summary) + + +if __name__ == "__main__": + main() diff --git a/source/conf.py b/source/conf.py index e2ed462c8b..bcc76b1b1f 100644 --- a/source/conf.py +++ b/source/conf.py @@ -175,6 +175,10 @@ # Archived snapshots don't change, and the Wayback Machine rate-limits CI (429). r"https://web\.archive\.org/.*", ] +if _baseline := os.getenv("LINKCHECK_BASELINE"): + from scripts.linkcheck import baseline_patterns + + linkcheck_ignore.extend(baseline_patterns(pathlib.Path(_baseline))) linkcheck_retries = 2 linkcheck_timeout = 30 # Ignore anchors for common targets when we know they likely won't be found diff --git a/source/contribute.rst b/source/contribute.rst index 112643d98c..a4560f156d 100644 --- a/source/contribute.rst +++ b/source/contribute.rst @@ -158,6 +158,41 @@ command: The guide will be browsable via http://localhost:8000. +Checking links +============== + +To check links locally, run: + +.. code-block:: bash + + nox -s linkcheck + +This checks all links, except those explicitly excluded in ``source/conf.py``. +Reports are written to ``build/output.txt`` and ``build/output.json``. + +In CI, pull requests and merge queues check external URLs that are absent from +the base revision's linkcheck report. New paths and fragments count as new URLs. +The base report is cached, or generated on demand if the cache is missing. +Local links are still checked. + +After pushes to ``main``, full linkcheck reports failures without failing the +aggregate ``check`` job. The daily run at 06:00 UTC checks all links and fails on +broken links. Both runs upload reports and list failing URLs in the job summary. + + +Testing the linkcheck tooling +============================= + +When changing the linkcheck tooling, run its pytest suite: + +.. code-block:: bash + + nox -s tests + +Nox installs the test dependencies automatically. The tests use a local HTTP +server and temporary Git repositories. They also run in CI. + + Where the guide is deployed =========================== diff --git a/tests/test_linkcheck.py b/tests/test_linkcheck.py new file mode 100644 index 0000000000..8ef2af4324 --- /dev/null +++ b/tests/test_linkcheck.py @@ -0,0 +1,255 @@ +import json +import re +import subprocess +import sys +from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from threading import Thread + +import pytest + +from scripts.linkcheck import ( + baseline_patterns, + prepare_baseline, + read_report, + summarize, +) + +ROOT = Path(__file__).resolve().parents[1] + + +def write_report(path, records): + path.write_text("".join(json.dumps(record) + "\n" for record in records)) + + +@pytest.mark.parametrize("status", ["working", "broken", "redirected", "ignored"]) +def test_exact_external_urls(tmp_path, status): + url = "https://example.org/docs_(v1)?a=1&b=2" + report = tmp_path / "output.json" + write_report( + report, + [ + {"uri": url, "status": status}, + {"uri": url, "status": status}, + {"uri": "/local/file", "status": "working"}, + {"uri": "mailto:docs@example.org", "status": "unchecked"}, + ], + ) + patterns = baseline_patterns(report) + assert len(patterns) == 1 + assert re.match(patterns[0], url) + for other in ( + url + "#missing-anchor", + url + "&new=value", + url + "/new-page", + url.replace("example.org", "exampleXorg"), + url + "\n", + "/local/file", + ): + assert not re.match(patterns[0], other) + + +@pytest.mark.parametrize( + "content", ["not JSON", "[]", "null", "{}", '{"uri": 1, "status": "broken"}'] +) +def test_invalid_reports(tmp_path, content): + report = tmp_path / "output.json" + report.write_text(content) + with pytest.raises(ValueError, match="output.json:1:"): + read_report(report) + + +def test_missing_and_empty_reports(tmp_path): + report = tmp_path / "output.json" + with pytest.raises(FileNotFoundError): + read_report(report) + report.write_text("\n") + assert baseline_patterns(report) == [] + + +def test_summary(tmp_path): + report, summary = tmp_path / "output.json", tmp_path / "summary.md" + write_report( + report, + [ + {"uri": "https://example.org/ok", "status": "working"}, + {"uri": "https://example.org/", "status": "broken", "info": "404"}, + {"uri": "https://example.org/slow", "status": "timeout"}, + ], + ) + summary.write_text("Existing summary\n") + summarize(report, summary) + text = summary.read_text() + assert text.startswith("Existing summary\n") + assert "2 failing URLs" in text + assert "<bad>" in text + assert "https://example.org/slow" in text + assert "https://example.org/ok" not in text + + +@pytest.fixture +def server(): + requests = [] + + class Handler(BaseHTTPRequestHandler): + def do_HEAD(self): + requests.append(self.path) + self.send_response(200 if self.path.startswith("/ok") else 404) + self.end_headers() + + def log_message(self, *args): + pass + + with ThreadingHTTPServer(("127.0.0.1", 0), Handler) as http: + thread = Thread(target=http.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{http.server_port}", requests + finally: + http.shutdown() + thread.join() + + +def documentation(directory, text, *, base=False): + source = directory / "source" + source.mkdir() + configuration = ( + "import os, sys\n" + f"sys.path.insert(0, {str(ROOT)!r})\n" + "from pathlib import Path\n" + "from scripts.linkcheck import baseline_patterns\n" + "linkcheck_ignore = []\n" + "if baseline := os.getenv('LINKCHECK_BASELINE'):\n" + " linkcheck_ignore = baseline_patterns(Path(baseline))\n" + "linkcheck_retries = 1\n" + "linkcheck_timeout = 2\n" + ) + if base: + configuration += ( + "assert not os.getenv('LINKCHECK_BASELINE')\n" + "assert not os.getenv('LINKCHECK_BASE_REF')\n" + ) + (source / "conf.py").write_text(configuration) + (source / "index.rst").write_text("Links\n=====\n\n" + text) + return source + + +def sphinx(source, output): + return subprocess.run( + [ + sys.executable, + "-m", + "sphinx", + "-q", + "-b", + "linkcheck", + "-d", + str(output / "doctrees"), + str(source), + str(output), + ], + capture_output=True, + text=True, + check=False, + ) + + +def test_sphinx_checks_only_new_urls(tmp_path, server, monkeypatch): + url, requests = server + monkeypatch.delenv("LINKCHECK_BASELINE", raising=False) + source = documentation(tmp_path, f"{url}/old-broken\n\n{url}/ok\n") + full = tmp_path / "full" + assert sphinx(source, full).returncode == 1 + assert "/old-broken" in requests + + monkeypatch.setenv("LINKCHECK_BASELINE", str(full / "output.json")) + requests.clear() + with (source / "index.rst").open("a") as stream: + stream.write(f"\n{url}/ok?new\n") + result = sphinx(source, tmp_path / "filtered") + assert result.returncode == 0, result.stdout + result.stderr + assert requests == ["/ok?new"] + + requests.clear() + with (source / "index.rst").open("a") as stream: + stream.write(f"\n{url}/new-broken\n") + assert sphinx(source, tmp_path / "new-broken").returncode == 1 + assert "/new-broken" in requests + assert "/old-broken" not in requests + + +def git(*args): + return subprocess.check_output(["git", *args], text=True).strip() + + +@contextmanager +def repository(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + git("init", "-q") + git("add", "source") + git( + "-c", + "user.name=Test", + "-c", + "user.email=test@example.org", + "commit", + "-qm", + "Docs", + ) + yield git("rev-parse", "HEAD") + assert git("worktree", "list", "--porcelain").count("worktree ") == 1 + + +def test_generate_baseline_and_reuse_cache(tmp_path, server, monkeypatch): + url, requests = server + documentation(tmp_path, f"{url}/old-broken\n", base=True) + report = tmp_path / "cache" / "baseline.jsonl" + with repository(tmp_path, monkeypatch) as base: + monkeypatch.setenv("LINKCHECK_BASELINE", "must-not-be-read") + monkeypatch.setenv("LINKCHECK_BASE_REF", "must-not-be-read") + prepare_baseline(base, report) + assert read_report(report)[0]["uri"] == f"{url}/old-broken" + assert "/old-broken" in requests + requests.clear() + prepare_baseline(base, report) + assert not requests + + +def test_baseline_failure_cleans_up_worktree(tmp_path, monkeypatch): + source = documentation(tmp_path, "No links.\n") + (source / "conf.py").write_text("raise RuntimeError('broken configuration')\n") + with repository(tmp_path, monkeypatch) as base: + report = tmp_path / "baseline.jsonl" + with pytest.raises(subprocess.CalledProcessError): + prepare_baseline(base, report) + assert not report.exists() + + +def test_fetch_baseline_in_shallow_checkout(tmp_path, server, monkeypatch): + url, _ = server + original = tmp_path / "original" + original.mkdir() + source = documentation(original, f"{url}/old-broken\n", base=True) + with repository(original, monkeypatch) as base: + (source / "index.rst").write_text("Current\n=======\n\nNo links.\n") + git("add", "source") + git( + "-c", + "user.name=Test", + "-c", + "user.email=test@example.org", + "commit", + "-qm", + "Change docs", + ) + checkout = tmp_path / "shallow" + git("clone", "--depth=1", original.as_uri(), str(checkout)) + with monkeypatch.context() as context: + context.chdir(checkout) + with pytest.raises(subprocess.CalledProcessError): + git("cat-file", "-e", base) + report = tmp_path / "baseline.jsonl" + prepare_baseline(base, report) + assert read_report(report)[0]["uri"] == f"{url}/old-broken" + assert git("worktree", "list", "--porcelain").count("worktree ") == 1