Merge pull request #22647 from github/rust/rust-analyzer-update-docs

Rust: Update rust-analyzer update instructions and update script

Author: paldepind

.github/workflows/update-rust-analyzer.lock.yml

@@ -75,6 +75,14 @@ safe-outputs:
       - "ruby/extractor/Cargo.toml"
       - "unified/extractor/Cargo.toml"
       - "unified/swift-syntax-rs/Cargo.toml"
+      # A rust-analyzer update can require a newer Rust toolchain whose Clippy
+      # lints require compatibility fixes in any of the Rust sources.
+      # rust-analyzer dependency updates can also update tree-sitter APIs,
+      # requiring compatibility fixes in the shared extractor implementation.
+      - "shared/yeast/src/**"
+      - "shared/yeast-macros/src/**"
+      - "shared/yeast-schema/src/**"
+      - "shared/tree-sitter-extractor/src/**"
       - "MODULE.bazel"
       - "MODULE.bazel.lock"
       - "rust-toolchain.toml"

.github/workflows/update-rust-analyzer.md

@@ -167,13 +167,16 @@ use_repo(
 
 http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
-# rust-analyzer sources needed by the rust ast-generator (see `rust/ast-generator/README.md`)
+# rust-analyzer sources needed by the rust ast-generator (see `rust/ast-generator/README.md`).
+# These should be updated through the `rust/scripts/update_rust_analyzer.py` script.
 RUST_ANALYZER_SRC_TAG = "2026-09-14"
 
+RUST_ANALYZER_SRC_INTEGRITY = "sha256-X08+CNiaj0xCaqv01pODxfdcroEMKuhsg2CvBQmWldU="
+
 http_archive(
     name = "rust-analyzer-src",
     build_file = "//rust/ast-generator:BUILD.rust-analyzer-src.bazel",
-    integrity = "sha256-X08+CNiaj0xCaqv01pODxfdcroEMKuhsg2CvBQmWldU=",
+    integrity = RUST_ANALYZER_SRC_INTEGRITY,
     patch_args = ["-p1"],
     patches = [
         "//rust/ast-generator:patches/rust-analyzer.patch",

MODULE.bazel

@@ -1,6 +1,9 @@
 #!/usr/bin/env python3
 
+import base64
+import hashlib
 import json
+import re
 import shlex
 import subprocess
 import sys
@@ -10,6 +13,7 @@
 
 
 REPO_ROOT = Path(__file__).resolve().parents[2]
+MODULE_BAZEL = REPO_ROOT / "MODULE.bazel"
 RUST_EXTRACTOR_MANIFEST = REPO_ROOT / "rust/extractor/Cargo.toml"
 
 # The files that mention the fixed Rust toolchain version
@@ -32,7 +36,7 @@ def run_codegen() -> None:
         run("bazel", "run", "//rust/codegen")
     except subprocess.CalledProcessError as error:
         print(
-            "\nCodegen failed. Carry out the instructions from step 4 in rust/updating-rust-analyzer.md manually.",
+            "\nCodegen failed. Carry out the instructions from step 5 in rust/updating-rust-analyzer.md manually.",
             file=sys.stderr,
             flush=True,
         )
@@ -61,12 +65,50 @@ def fetch(url: str) -> bytes:
         return response.read()
 
 
-def get_compatible_rust_toolchain(rust_analyzer_version: str) -> str:
-    """Get the latest Rust toolchain released no later than rust-analyzer."""
-    # Get the release date of the rust-analyzer version
+def get_rust_analyzer_release_date(rust_analyzer_version: str) -> str:
+    """Get the release date of a rust-analyzer crate version."""
     crate_url = f"https://crates.io/api/v1/crates/ra_ap_syntax/{rust_analyzer_version}"
     crate = json.loads(fetch(crate_url))
-    rust_analyzer_release = crate["version"]["created_at"].split("T")[0]
+    return crate["version"]["created_at"].split("T")[0]
+
+
+def update_rust_analyzer_sources(rust_analyzer_version: str) -> None:
+    """
+    Update the rust-analyzer source archive used by the AST generator.
+
+    Concretely, this updates `RUST_ANALYZER_SRC_TAG` and
+    `RUST_ANALYZER_SRC_INTEGRITY` in the MODULE.bazel file.
+    """
+
+    release_date = get_rust_analyzer_release_date(rust_analyzer_version)
+    archive_url = (
+        "https://github.com/rust-lang/rust-analyzer/archive/refs/tags/"
+        f"{release_date}.tar.gz"
+    )
+    archive = fetch(archive_url)
+    integrity = "sha256-" + base64.b64encode(hashlib.sha256(archive).digest()).decode()
+
+    module = MODULE_BAZEL.read_text()
+    module, tag_replacements = re.subn(
+        r'RUST_ANALYZER_SRC_TAG = "[^"]+"',
+        f'RUST_ANALYZER_SRC_TAG = "{release_date}"',
+        module,
+    )
+    if tag_replacements != 1:
+        raise RuntimeError("expected exactly one RUST_ANALYZER_SRC_TAG assignment")
+    module, integrity_replacements = re.subn(
+        r'RUST_ANALYZER_SRC_INTEGRITY = "[^"]+"',
+        f'RUST_ANALYZER_SRC_INTEGRITY = "{integrity}"',
+        module,
+    )
+    if integrity_replacements != 1:
+        raise RuntimeError("expected exactly one RUST_ANALYZER_SRC_INTEGRITY assignment")
+    MODULE_BAZEL.write_text(module)
+
+
+def get_compatible_rust_toolchain(rust_analyzer_version: str) -> str:
+    """Get the latest Rust toolchain released no later than rust-analyzer."""
+    rust_analyzer_release = get_rust_analyzer_release_date(rust_analyzer_version)
 
     # `manifests.txt` is a list of all toolchains. The one we're interested in looks like
     # ```
@@ -168,20 +210,24 @@ def main() -> None:
         run("cargo", "update")
     commit_all("Cargo: Upgrade dependencies")
 
-    print_step(2, "Update the fixed Rust toolchain used by the extractor")
+    print_step(2, "Update the rust-analyzer sources used by the AST generator")
+    update_rust_analyzer_sources(new_rust_analyzer_version)
+    commit_all("Rust: Update rust-analyzer sources")
+
+    print_step(3, "Update the fixed Rust toolchain used by the extractor")
     rust_toolchain = get_compatible_rust_toolchain(new_rust_analyzer_version)
     update_fixed_rust_toolchain_versions(rust_toolchain)
     commit_all("Rust: Update fixed toolchain")
 
-    print_step(3, "Regenerate vendored bazel files")
+    print_step(4, "Regenerate vendored bazel files")
     run("misc/bazel/3rdparty/update_tree_sitter_extractors_deps.sh")
     commit_all("Bazel: Regenerate vendored cargo dependencies")
 
-    print_step(4, "Run codegen")
+    print_step(5, "Run codegen")
     run_codegen()
     commit_all("Rust: Run codegen")
 
-    print_step(5, "Try compiling")
+    print_step(6, "Try compiling")
     run("bazel", "run", "//rust:install")