deps: move tree-sitter-language-pack to 1.x, with a clear error when a grammar cannot be downloaded

CHANGELOG.md

@@ -93,6 +93,17 @@ uses [Semantic Versioning](https://semver.org/).
   `--update` keeps it, so existing flat docs stay flat. The GitHub Pages
   viewer, the web app and the MCP tools (`doc_path` in
   `processing_order.json`) follow the layout.
+- **tree-sitter-language-pack 1.x.** The pin moves from `>=0.8.0,<0.9.0` to
+  `>=1.23.1,<2.0.0`, which brings grammars 0.8 does not ship (Lean, among
+  others). 1.x downloads a grammar on first use and caches it under
+  `tree_sitter_language_pack.cache_dir()`, so the first run of the Dart, Go,
+  JavaScript/TypeScript-in-Vue, Vue and Zig analyzers needs network access;
+  the Docker image fetches them at build time. Without network access and
+  without a cached grammar, the analysis stops before the first file with one
+  error that names the grammar, the cache folder and the `t.download([...])`
+  command, instead of a traceback per file. The 1.x Dart grammar parses
+  `Foo<T>(x)` as a call instead of a relational expression, and the Dart
+  analyzer now takes the type-argument edges from that call too.
 
 ## [2.0.0] - 2026-09-18
 

codewiki/src/be/dependency_analyzer/analysis/call_graph_analyzer.py

@@ -22,6 +22,7 @@
     is_external_symbol,
     is_macro_name,
 )
+from codewiki.src.be.dependency_analyzer.utils.grammars import ensure_pack_grammars
 from codewiki.src.be.dependency_analyzer.utils.patterns import CODE_EXTENSIONS
 from codewiki.src.be.dependency_analyzer.utils.security import safe_open_text
 
@@ -96,6 +97,9 @@ def analyze_code_files(self, code_files: list[dict], base_dir: str) -> dict:
         self.functions = {}
         self.call_relationships = []
         code_files = self._route_contextual_headers(code_files, base_dir)
+        # Fail once, with a clear message, when a pack grammar cannot be
+        # downloaded, instead of once per file inside the per-file handlers.
+        ensure_pack_grammars(f["language"] for f in code_files)
         self._python_project_modules = self._collect_python_modules(code_files)
         self._python_external_import_roots = set()
         self._dart_directives = {}

codewiki/src/be/dependency_analyzer/analyzers/dart.py

@@ -36,7 +36,7 @@
 from functools import lru_cache
 from pathlib import Path
 
-from tree_sitter_language_pack import get_parser
+from codewiki.src.be.dependency_analyzer.utils.grammars import get_pack_parser
 
 from codewiki.src.be.dependency_analyzer.analyzers.dart_flutter import (
     COMPOSITION_METHOD_RE,
@@ -212,7 +212,7 @@ def _local_declared_types(*roots) -> dict[str, str]:
 
 @lru_cache(maxsize=1)
 def _dart_parser():
-    return get_parser("dart")
+    return get_pack_parser("dart")
 
 
 def _text(node) -> str:
@@ -762,6 +762,17 @@ def _process_chain(self, target, head, selectors, local_types, lift_to):
             return
         caller, owner = target.caller, target.owner
         line = head.start_point[0] + 1
+        # `Foo<T>(x)` / `foo<T>(x)`: the type arguments sit in the call's
+        # argument_part (grammars that misparse it as a relational expression
+        # are handled by _handle_generic_invocation).
+        for token_kind, _, token_node in tokens:
+            if token_kind != "call":
+                continue
+            type_args = _first_child(token_node, "type_arguments")
+            if type_args is not None:
+                for ref in _descendants(type_args):
+                    if ref.type == "type_identifier":
+                        self._add_type_edge(caller, _text(ref), line)
         name = _text(head)
         if head.type == "identifier" and name in self.import_prefixes and tokens[0][0] == "member":
             name, tokens = tokens[0][1] or "", tokens[1:]

codewiki/src/be/dependency_analyzer/analyzers/go.py

@@ -48,7 +48,7 @@
 from functools import lru_cache
 from pathlib import Path
 
-from tree_sitter_language_pack import get_parser
+from codewiki.src.be.dependency_analyzer.utils.grammars import get_pack_parser
 
 from codewiki.src.be.dependency_analyzer.models.core import CallRelationship, Node
 from codewiki.src.be.dependency_analyzer.utils.file_selection import is_skipped_go_file
@@ -326,7 +326,7 @@ def _default_import_names(import_path: str) -> set[str]:
 
 @lru_cache(maxsize=1)
 def _go_parser():
-    return get_parser("go")
+    return get_pack_parser("go")
 
 
 class TreeSitterGoAnalyzer:

codewiki/src/be/dependency_analyzer/analyzers/vue.py

@@ -37,7 +37,7 @@
 from functools import lru_cache
 from pathlib import Path
 
-from tree_sitter_language_pack import get_parser
+from codewiki.src.be.dependency_analyzer.utils.grammars import get_pack_parser
 
 from codewiki.src.be.dependency_analyzer.models.core import CallRelationship, Node
 
@@ -107,7 +107,7 @@
 
 @lru_cache(maxsize=None)
 def _parser(language: str):
-    return get_parser(language)
+    return get_pack_parser(language)
 
 
 def _is_sfc(target: str) -> bool:

codewiki/src/be/dependency_analyzer/analyzers/zig.py

@@ -47,7 +47,7 @@
 from functools import lru_cache
 from pathlib import Path
 
-from tree_sitter_language_pack import get_parser
+from codewiki.src.be.dependency_analyzer.utils.grammars import get_pack_parser
 
 from codewiki.src.be.dependency_analyzer.models.core import CallRelationship, Node
 from codewiki.src.be.dependency_analyzer.utils.file_selection import is_skipped_zig_file
@@ -107,7 +107,7 @@ def _build_zig_modules(repo_path: str) -> dict[str, str]:
 
 
 def _zig_parser():
-    return get_parser("zig")
+    return get_pack_parser("zig")
 
 
 def _alias_chain(var_decl) -> tuple[tuple[str, str], tuple[str, ...]] | None:

codewiki/src/be/dependency_analyzer/utils/grammars.py

@@ -0,0 +1,63 @@
+"""Parsers from ``tree-sitter-language-pack``.
+
+Pack 1.x does not ship its grammars in the wheel. It downloads each grammar
+the first time it is loaded and caches it under ``cache_dir()``. Without
+network access that first load fails with a ``DownloadError`` that names a
+URL but not the cause or the fix. The per-file ``except`` blocks of the
+analyzers would also swallow it, and the language would silently drop out of
+the graph. So the analysis checks the grammars it needs once, before it
+parses any file, and stops with one clear message.
+"""
+
+from functools import lru_cache
+
+import tree_sitter_language_pack as tslp
+
+# Pack grammars each analyzed language loads. ``docker/Dockerfile`` downloads
+# the same set at build time (``tests/test_pack_grammars.py`` checks this).
+PACK_GRAMMARS_BY_LANGUAGE: dict[str, tuple[str, ...]] = {
+    "dart": ("dart",),
+    "go": ("go",),
+    "vue": ("vue", "javascript", "typescript"),
+    "zig": ("zig",),
+}
+
+PACK_GRAMMARS: tuple[str, ...] = tuple(
+    sorted({g for grammars in PACK_GRAMMARS_BY_LANGUAGE.values() for g in grammars})
+)
+
+
+class GrammarUnavailableError(RuntimeError):
+    """A pack grammar is not cached and could not be downloaded."""
+
+    def __init__(self, grammar: str, cause: Exception):
+        try:
+            cache = tslp.cache_dir()
+        except Exception:  # noqa: BLE001 — the path only improves the message
+            cache = "the tree-sitter-language-pack cache"
+        names = ", ".join(repr(g) for g in PACK_GRAMMARS)
+        super().__init__(
+            f"The tree-sitter grammar '{grammar}' is not in {cache} and could not be "
+            f"downloaded ({cause}). tree-sitter-language-pack downloads each grammar on "
+            "first use, so the first analysis of these files needs network access. "
+            "To prepare a machine without network access, run this once on a machine "
+            "with access and copy the cache folder: "
+            f'python -c "import tree_sitter_language_pack as t; t.download([{names}])"'
+        )
+        self.grammar = grammar
+
+
+@lru_cache(maxsize=None)
+def get_pack_parser(grammar: str):
+    """Return the pack parser for ``grammar``, with a clear error when it is unavailable."""
+    try:
+        return tslp.get_parser(grammar)
+    except tslp.DownloadError as e:
+        raise GrammarUnavailableError(grammar, e) from e
+
+
+def ensure_pack_grammars(languages) -> None:
+    """Load every pack grammar that ``languages`` need, or raise ``GrammarUnavailableError``."""
+    for language in sorted(set(languages)):
+        for grammar in PACK_GRAMMARS_BY_LANGUAGE.get(language, ()):
+            get_pack_parser(grammar)

docker/Dockerfile

@@ -18,6 +18,10 @@ COPY requirements.txt .
 # Install Python dependencies
 RUN pip install --no-cache-dir -r requirements.txt
 
+# tree-sitter-language-pack 1.x downloads each grammar on first use. Fetch the
+# ones the analyzers load at build time, so the image runs offline.
+RUN python -c "import tree_sitter_language_pack as t; t.download(['dart', 'go', 'javascript', 'typescript', 'vue', 'zig'])"
+
 # Copy application code
 COPY codewiki ./codewiki
 COPY img ./img

guides/development.md

@@ -122,7 +122,12 @@ Analyzers are standalone classes, one file per language. Follow
 1. **Grammar.** Add the `tree-sitter-<lang>` package to `pyproject.toml`
    and `requirements.txt`, or take the grammar from
    `tree-sitter-language-pack` (already a dependency) with
-   `get_parser("<lang>")`, as `analyzers/dart.py` and `analyzers/go.py` do.
+   `get_pack_parser("<lang>")` from `utils/grammars.py`, as
+   `analyzers/dart.py` and `analyzers/go.py` do. Add the language and its
+   grammars to `PACK_GRAMMARS_BY_LANGUAGE` in the same file. The pack (1.x) downloads a grammar on first
+   use, so also add the name to the `t.download([...])` list in
+   `docker/Dockerfile`. `tests/test_pack_grammars.py` checks that the two
+   lists match.
 2. **Analyzer.** Create `analyzers/<lang>.py` with a
    `TreeSitter<Lang>Analyzer` class that extracts components and
    relationships, plus a module-level `analyze_<lang>_file(...)` function.

pyproject.toml

@@ -29,7 +29,7 @@ dependencies = [
     "GitPython>=3.1.40",
     "Jinja2>=3.1.6",
     "tree-sitter>=0.25",
-    "tree-sitter-language-pack>=0.8.0,<0.9.0",
+    "tree-sitter-language-pack>=1.23.1,<2.0.0",
     "tree-sitter-python>=0.23.6",
     "tree-sitter-java>=0.23.5",
     "tree-sitter-javascript>=0.21.4",

requirements.txt

@@ -148,7 +148,7 @@ tree-sitter-embedded-template==0.23.2
 tree-sitter-java==0.23.5
 tree-sitter-javascript==0.21.4
 tree-sitter-kotlin==1.1.0
-tree-sitter-language-pack==0.8.0
+tree-sitter-language-pack==1.23.1
 tree-sitter-python==0.23.6
 tree-sitter-ruby==0.23.1
 tree-sitter-rust==0.23.2

tests/test_pack_grammars.py

@@ -0,0 +1,80 @@
+"""Grammars from tree-sitter-language-pack: a failed download stops the
+analysis once, with a clear message, and the Docker image fetches every
+grammar the analyzers load."""
+
+import re
+from pathlib import Path
+
+import pytest
+import tree_sitter_language_pack as tslp
+
+from codewiki.src.be.dependency_analyzer.analysis.call_graph_analyzer import (
+    CallGraphAnalyzer,
+)
+from codewiki.src.be.dependency_analyzer.utils import grammars
+from codewiki.src.be.dependency_analyzer.utils.grammars import (
+    PACK_GRAMMARS,
+    GrammarUnavailableError,
+    ensure_pack_grammars,
+    get_pack_parser,
+)
+
+DOCKERFILE = Path(__file__).resolve().parents[1] / "docker" / "Dockerfile"
+
+
+@pytest.fixture
+def offline(monkeypatch):
+    """Every pack grammar load fails as it does without network access."""
+    calls = []
+
+    def fail(name):
+        calls.append(name)
+        raise tslp.DownloadError(f"Failed to fetch manifest for {name}: Connection refused")
+
+    get_pack_parser.cache_clear()
+    monkeypatch.setattr(grammars.tslp, "get_parser", fail)
+    yield calls
+    get_pack_parser.cache_clear()
+
+
+def test_download_failure_gives_a_clear_error(offline):
+    with pytest.raises(GrammarUnavailableError) as info:
+        get_pack_parser("go")
+    message = str(info.value)
+    assert info.value.grammar == "go"
+    assert "'go'" in message
+    assert "network access" in message
+    assert "Connection refused" in message
+    assert "t.download([" in message
+    assert isinstance(info.value.__cause__, tslp.DownloadError)
+
+
+def test_only_the_grammars_of_present_languages_load(offline, monkeypatch):
+    monkeypatch.setattr(grammars.tslp, "get_parser", lambda name: offline.append(name))
+    ensure_pack_grammars(["python", "vue", "java", "vue"])
+    assert sorted(offline) == ["javascript", "typescript", "vue"]
+
+
+def test_languages_without_pack_grammars_need_no_network(offline):
+    ensure_pack_grammars(["python", "java", "rust"])
+    assert offline == []
+
+
+def test_analysis_stops_before_the_first_file(offline, tmp_path, monkeypatch):
+    (tmp_path / "main.go").write_text("package main\n\nfunc main() {}\n", encoding="utf-8")
+    analyzed = []
+    monkeypatch.setattr(
+        CallGraphAnalyzer, "_analyze_code_file", lambda self, base, info: analyzed.append(info)
+    )
+    with pytest.raises(GrammarUnavailableError, match="'go'"):
+        CallGraphAnalyzer().analyze_code_files(
+            [{"path": "main.go", "language": "go"}], str(tmp_path)
+        )
+    assert analyzed == []
+
+
+def test_dockerfile_downloads_every_pack_grammar():
+    match = re.search(r"t\.download\(\[([^\]]*)\]\)", DOCKERFILE.read_text(encoding="utf-8"))
+    assert match, "docker/Dockerfile must download the pack grammars at build time"
+    in_dockerfile = sorted(re.findall(r"'([^']+)'", match.group(1)))
+    assert in_dockerfile == list(PACK_GRAMMARS)