workflows: retry transient Bazel download failures in setup-bazel

.github/actions/setup-bazel/BUILD.bazel

@@ -0,0 +1,12 @@
+package(default_visibility = ["//tests:__subpackages__"])
+
+exports_files(
+    glob(["*"]),
+    visibility = ["//tests:__subpackages__"],
+)
+
+filegroup(
+    name = "all_files",
+    srcs = glob(["*"]),
+    visibility = ["//tests:__subpackages__"],
+)

.github/actions/setup-bazel/action.yaml

@@ -5,8 +5,11 @@ runs:
   using: "composite"
   steps:
     - name: Setup Bazel
-      uses: bazel-contrib/setup-bazel@0.19.0
+      uses: bazel-contrib/setup-bazel@0.19.1
       with:
+        # Windows images have a fixed-version `bazel` on PATH, and Bazelisk
+        # < 1.29.0 has no windows-arm64 build.
+        bazelisk-version: 1.29.0
         bazelisk-cache: true
         disk-cache: true
         # Files hashed to compute the repository cache key. If only `true` is
@@ -23,3 +26,11 @@ runs:
           # pyrefly toolchain or type-check targets before running them.
           build --output_groups=-pyrefly
           build --//python/config_settings:py_pbs_distribution=install_only_stripped
+
+    # setup-bazel only configures Bazelisk and restores caches; the Bazel
+    # binary itself is downloaded by Bazelisk on first invocation. Because
+    # Bazelisk does not retry mid-stream download failures, trigger and retry
+    # the download upfront via `bazel --version`.
+    - name: Download Bazel
+      shell: bash
+      run: "${GITHUB_ACTION_PATH}/download_bazel.sh"

.github/actions/setup-bazel/download_bazel.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# Downloads the Bazel binary via Bazelisk with retries on transient network
+# errors.
+#
+# `bazel-contrib/setup-bazel` only configures Bazelisk and restores caches; the
+# Bazel binary itself is downloaded by Bazelisk on first invocation. Bazelisk
+# retries initial HTTP GET headers (`client.Do`), but does not retry mid-stream
+# transfer errors during `io.Copy` (such as HTTP/2 `INTERNAL_ERROR` stream
+# resets from `releases.bazel.build`). Running `bazel --version` here triggers
+# the download upfront (without starting the Bazel server JVM) and retries on
+# transient failures.
+set -euo pipefail
+
+max_attempts="${BAZEL_DOWNLOAD_MAX_ATTEMPTS:-5}"
+delay="${BAZEL_DOWNLOAD_INITIAL_DELAY:-2}"
+
+for ((attempt = 1; attempt <= max_attempts; attempt++)); do
+  if bazel --version; then
+    exit 0
+  fi
+  if ((attempt < max_attempts)); then
+    echo "::warning::Failed to download Bazel (attempt $attempt/$max_attempts); retrying in ${delay}s..."
+    sleep "$delay"
+    delay=$((delay * 2))
+  fi
+done
+
+echo "::error::Failed to download Bazel after $max_attempts attempts"
+# Remove any empty/partial Bazelisk cache directory created before the failed
+# download so `setup-bazel`'s post step does not cache an empty directory.
+if [[ -n "${BAZELISK_HOME:-}" ]]; then
+  rm -rf "$BAZELISK_HOME"
+fi
+rm -rf \
+  "${XDG_CACHE_HOME:-$HOME/.cache}/bazelisk" \
+  "$HOME/Library/Caches/bazelisk" \
+  "$HOME/AppData/Local/bazelisk"
+exit 1

.github/workflows/release_build_artifacts.yaml

@@ -81,11 +81,7 @@ jobs:
           persist-credentials: false
 
       - name: Setup Bazel
-        uses: bazel-contrib/setup-bazel@0.19.1
-        with:
-          # Windows images also have a fixed-version `bazel` on PATH. 1.20.0,
-          # used by the other workflows, has no windows-arm64 build.
-          bazelisk-version: 1.29.0
+        uses: ./.github/actions/setup-bazel
 
       # Release commands run main's workflows, even for patch releases of older
       # release branches. Skip refs without dev/platforms, since their scripts

tests/workflows/BUILD.bazel

@@ -75,3 +75,16 @@ pytest_test(
         "//python/runfiles",
     ],
 )
+
+pytest_test(
+    name = "setup_bazel_test",
+    srcs = ["setup_bazel_test.py"],
+    data = [
+        "//.github/actions/setup-bazel:all_files",
+    ],
+    target_compatible_with = NOT_WINDOWS,
+    deps = [
+        "//python/runfiles",
+        "@pypi//pyyaml",
+    ],
+)

tests/workflows/setup_bazel_test.py

@@ -0,0 +1,145 @@
+"""Tests for the .github/actions/setup-bazel composite action."""
+
+from __future__ import annotations
+
+import os
+import stat
+import subprocess
+from pathlib import Path
+
+import yaml
+
+from python.runfiles import runfiles
+
+
+def _action_dir() -> Path:
+    return runfiles.CreateOrRaise().root() / "rules_python/.github/actions/setup-bazel"
+
+
+def _write_fake_bazel(bin_dir: Path, body: str) -> Path:
+    bin_dir.mkdir(parents=True, exist_ok=True)
+    fake_bazel = bin_dir / "bazel"
+    fake_bazel.write_text(
+        f"#!/usr/bin/env bash\nset -euo pipefail\n{body}\n", encoding="utf-8"
+    )
+    fake_bazel.chmod(0o755)
+    return fake_bazel
+
+
+def test_action_yaml_invokes_executable_download_script():
+    """Verifies setup-bazel action.yaml runs download_bazel.sh and the script is +x."""
+    action_dir = _action_dir()
+    script = action_dir / "download_bazel.sh"
+    assert script.is_file()
+    assert bool(script.stat().st_mode & stat.S_IXUSR), (
+        "download_bazel.sh must be executable"
+    )
+
+    action = yaml.safe_load((action_dir / "action.yaml").read_text(encoding="utf-8"))
+    steps = action["runs"]["steps"]
+    assert any("download_bazel.sh" in step.get("run", "") for step in steps)
+
+
+def test_download_bazel_succeeds_first_try(tmp_path: Path):
+    """Verifies download_bazel.sh succeeds without warnings when bazel --version succeeds."""
+    script = _action_dir() / "download_bazel.sh"
+    calls_file = tmp_path / "calls.txt"
+    _write_fake_bazel(
+        tmp_path / "bin",
+        f'echo "$@" >> "{calls_file}"\necho "bazel 9.1.1"\n',
+    )
+
+    env = os.environ.copy()
+    env["PATH"] = f"{tmp_path / 'bin'}:{env.get('PATH', '')}"
+    env["HOME"] = str(tmp_path / "home")
+
+    result = subprocess.run(
+        [str(script)],
+        env=env,
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    assert result.returncode == 0
+    assert calls_file.read_text(encoding="utf-8").splitlines() == ["--version"]
+    assert "::warning::" not in result.stdout
+    assert "::error::" not in result.stdout
+
+
+def test_download_bazel_retries_transient_failure(tmp_path: Path):
+    """Verifies download_bazel.sh retries transient failures and succeeds."""
+    script = _action_dir() / "download_bazel.sh"
+    calls_file = tmp_path / "calls.txt"
+    _write_fake_bazel(
+        tmp_path / "bin",
+        f"""
+echo "$@" >> "{calls_file}"
+count=$(wc -l < "{calls_file}")
+if (( count < 3 )); then
+  echo "could not download Bazel: stream error: INTERNAL_ERROR" >&2
+  exit 1
+fi
+mkdir -p "$HOME/.cache/bazelisk/downloads"
+echo "bazel 9.1.1"
+""",
+    )
+
+    home = tmp_path / "home"
+    env = os.environ.copy()
+    env["PATH"] = f"{tmp_path / 'bin'}:{env.get('PATH', '')}"
+    env["HOME"] = str(home)
+    env["BAZEL_DOWNLOAD_INITIAL_DELAY"] = "0"
+
+    result = subprocess.run(
+        [str(script)],
+        env=env,
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    assert result.returncode == 0
+    assert calls_file.read_text(encoding="utf-8").splitlines() == [
+        "--version",
+        "--version",
+        "--version",
+    ]
+    assert "::warning::Failed to download Bazel (attempt 1/5)" in result.stdout
+    assert "::warning::Failed to download Bazel (attempt 2/5)" in result.stdout
+    assert (home / ".cache/bazelisk/downloads").is_dir()
+
+
+def test_download_bazel_exhausts_retries_and_cleans_cache(tmp_path: Path):
+    """Verifies download_bazel.sh cleans up partial cache dirs when all retries fail."""
+    script = _action_dir() / "download_bazel.sh"
+    calls_file = tmp_path / "calls.txt"
+    _write_fake_bazel(
+        tmp_path / "bin",
+        f"""
+echo "$@" >> "{calls_file}"
+mkdir -p "$HOME/.cache/bazelisk/downloads/_tmp"
+mkdir -p "$BAZELISK_HOME/downloads/_tmp"
+exit 1
+""",
+    )
+
+    home = tmp_path / "home"
+    bazelisk_home = tmp_path / "bazelisk_home"
+    env = os.environ.copy()
+    env["PATH"] = f"{tmp_path / 'bin'}:{env.get('PATH', '')}"
+    env["HOME"] = str(home)
+    env["BAZELISK_HOME"] = str(bazelisk_home)
+    env["BAZEL_DOWNLOAD_MAX_ATTEMPTS"] = "3"
+    env["BAZEL_DOWNLOAD_INITIAL_DELAY"] = "0"
+
+    result = subprocess.run(
+        [str(script)],
+        env=env,
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    assert result.returncode == 1
+    assert len(calls_file.read_text(encoding="utf-8").splitlines()) == 3
+    assert "::error::Failed to download Bazel after 3 attempts" in result.stdout
+    assert not (home / ".cache/bazelisk").exists()
+    assert not bazelisk_home.exists()