diff --git a/CHANGELOG.md b/CHANGELOG.md index 47d174ca..fc96c028 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -93,6 +93,17 @@ uses [Semantic Versioning](https://semver.org/). `--update` keeps it, so existing flat docs stay flat. The GitHub Pages viewer, the web app and the MCP tools (`doc_path` in `processing_order.json`) follow the layout. +- **tree-sitter-language-pack 1.x.** The pin moves from `>=0.8.0,<0.9.0` to + `>=1.23.1,<2.0.0`, which brings grammars 0.8 does not ship (Lean, among + others). 1.x downloads a grammar on first use and caches it under + `tree_sitter_language_pack.cache_dir()`, so the first run of the Dart, Go, + JavaScript/TypeScript-in-Vue, Vue and Zig analyzers needs network access; + the Docker image fetches them at build time. Without network access and + without a cached grammar, the analysis stops before the first file with one + error that names the grammar, the cache folder and the `t.download([...])` + command, instead of a traceback per file. The 1.x Dart grammar parses + `Foo(x)` as a call instead of a relational expression, and the Dart + analyzer now takes the type-argument edges from that call too. ## [2.0.0] - 2026-09-18 diff --git a/codewiki/src/be/dependency_analyzer/analysis/call_graph_analyzer.py b/codewiki/src/be/dependency_analyzer/analysis/call_graph_analyzer.py index c52d090f..58b21b22 100644 --- a/codewiki/src/be/dependency_analyzer/analysis/call_graph_analyzer.py +++ b/codewiki/src/be/dependency_analyzer/analysis/call_graph_analyzer.py @@ -22,6 +22,7 @@ is_external_symbol, is_macro_name, ) +from codewiki.src.be.dependency_analyzer.utils.grammars import ensure_pack_grammars from codewiki.src.be.dependency_analyzer.utils.patterns import CODE_EXTENSIONS from codewiki.src.be.dependency_analyzer.utils.security import safe_open_text @@ -96,6 +97,9 @@ def analyze_code_files(self, code_files: list[dict], base_dir: str) -> dict: self.functions = {} self.call_relationships = [] code_files = self._route_contextual_headers(code_files, base_dir) + # Fail once, with a clear message, when a pack grammar cannot be + # downloaded, instead of once per file inside the per-file handlers. + ensure_pack_grammars(f["language"] for f in code_files) self._python_project_modules = self._collect_python_modules(code_files) self._python_external_import_roots = set() self._dart_directives = {} diff --git a/codewiki/src/be/dependency_analyzer/analyzers/dart.py b/codewiki/src/be/dependency_analyzer/analyzers/dart.py index c773265d..6af1b0c9 100644 --- a/codewiki/src/be/dependency_analyzer/analyzers/dart.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/dart.py @@ -36,7 +36,7 @@ from functools import lru_cache from pathlib import Path -from tree_sitter_language_pack import get_parser +from codewiki.src.be.dependency_analyzer.utils.grammars import get_pack_parser from codewiki.src.be.dependency_analyzer.analyzers.dart_flutter import ( COMPOSITION_METHOD_RE, @@ -212,7 +212,7 @@ def _local_declared_types(*roots) -> dict[str, str]: @lru_cache(maxsize=1) def _dart_parser(): - return get_parser("dart") + return get_pack_parser("dart") def _text(node) -> str: @@ -762,6 +762,17 @@ def _process_chain(self, target, head, selectors, local_types, lift_to): return caller, owner = target.caller, target.owner line = head.start_point[0] + 1 + # `Foo(x)` / `foo(x)`: the type arguments sit in the call's + # argument_part (grammars that misparse it as a relational expression + # are handled by _handle_generic_invocation). + for token_kind, _, token_node in tokens: + if token_kind != "call": + continue + type_args = _first_child(token_node, "type_arguments") + if type_args is not None: + for ref in _descendants(type_args): + if ref.type == "type_identifier": + self._add_type_edge(caller, _text(ref), line) name = _text(head) if head.type == "identifier" and name in self.import_prefixes and tokens[0][0] == "member": name, tokens = tokens[0][1] or "", tokens[1:] diff --git a/codewiki/src/be/dependency_analyzer/analyzers/go.py b/codewiki/src/be/dependency_analyzer/analyzers/go.py index 7a241812..8c0042fa 100644 --- a/codewiki/src/be/dependency_analyzer/analyzers/go.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/go.py @@ -48,7 +48,7 @@ from functools import lru_cache from pathlib import Path -from tree_sitter_language_pack import get_parser +from codewiki.src.be.dependency_analyzer.utils.grammars import get_pack_parser from codewiki.src.be.dependency_analyzer.models.core import CallRelationship, Node from codewiki.src.be.dependency_analyzer.utils.file_selection import is_skipped_go_file @@ -326,7 +326,7 @@ def _default_import_names(import_path: str) -> set[str]: @lru_cache(maxsize=1) def _go_parser(): - return get_parser("go") + return get_pack_parser("go") class TreeSitterGoAnalyzer: diff --git a/codewiki/src/be/dependency_analyzer/analyzers/vue.py b/codewiki/src/be/dependency_analyzer/analyzers/vue.py index 1a17f1ab..4934f72c 100644 --- a/codewiki/src/be/dependency_analyzer/analyzers/vue.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/vue.py @@ -37,7 +37,7 @@ from functools import lru_cache from pathlib import Path -from tree_sitter_language_pack import get_parser +from codewiki.src.be.dependency_analyzer.utils.grammars import get_pack_parser from codewiki.src.be.dependency_analyzer.models.core import CallRelationship, Node @@ -107,7 +107,7 @@ @lru_cache(maxsize=None) def _parser(language: str): - return get_parser(language) + return get_pack_parser(language) def _is_sfc(target: str) -> bool: diff --git a/codewiki/src/be/dependency_analyzer/analyzers/zig.py b/codewiki/src/be/dependency_analyzer/analyzers/zig.py index cd5f1b7c..feb60e2e 100644 --- a/codewiki/src/be/dependency_analyzer/analyzers/zig.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/zig.py @@ -47,7 +47,7 @@ from functools import lru_cache from pathlib import Path -from tree_sitter_language_pack import get_parser +from codewiki.src.be.dependency_analyzer.utils.grammars import get_pack_parser from codewiki.src.be.dependency_analyzer.models.core import CallRelationship, Node from codewiki.src.be.dependency_analyzer.utils.file_selection import is_skipped_zig_file @@ -107,7 +107,7 @@ def _build_zig_modules(repo_path: str) -> dict[str, str]: def _zig_parser(): - return get_parser("zig") + return get_pack_parser("zig") def _alias_chain(var_decl) -> tuple[tuple[str, str], tuple[str, ...]] | None: diff --git a/codewiki/src/be/dependency_analyzer/utils/grammars.py b/codewiki/src/be/dependency_analyzer/utils/grammars.py new file mode 100644 index 00000000..76de670b --- /dev/null +++ b/codewiki/src/be/dependency_analyzer/utils/grammars.py @@ -0,0 +1,63 @@ +"""Parsers from ``tree-sitter-language-pack``. + +Pack 1.x does not ship its grammars in the wheel. It downloads each grammar +the first time it is loaded and caches it under ``cache_dir()``. Without +network access that first load fails with a ``DownloadError`` that names a +URL but not the cause or the fix. The per-file ``except`` blocks of the +analyzers would also swallow it, and the language would silently drop out of +the graph. So the analysis checks the grammars it needs once, before it +parses any file, and stops with one clear message. +""" + +from functools import lru_cache + +import tree_sitter_language_pack as tslp + +# Pack grammars each analyzed language loads. ``docker/Dockerfile`` downloads +# the same set at build time (``tests/test_pack_grammars.py`` checks this). +PACK_GRAMMARS_BY_LANGUAGE: dict[str, tuple[str, ...]] = { + "dart": ("dart",), + "go": ("go",), + "vue": ("vue", "javascript", "typescript"), + "zig": ("zig",), +} + +PACK_GRAMMARS: tuple[str, ...] = tuple( + sorted({g for grammars in PACK_GRAMMARS_BY_LANGUAGE.values() for g in grammars}) +) + + +class GrammarUnavailableError(RuntimeError): + """A pack grammar is not cached and could not be downloaded.""" + + def __init__(self, grammar: str, cause: Exception): + try: + cache = tslp.cache_dir() + except Exception: # noqa: BLE001 — the path only improves the message + cache = "the tree-sitter-language-pack cache" + names = ", ".join(repr(g) for g in PACK_GRAMMARS) + super().__init__( + f"The tree-sitter grammar '{grammar}' is not in {cache} and could not be " + f"downloaded ({cause}). tree-sitter-language-pack downloads each grammar on " + "first use, so the first analysis of these files needs network access. " + "To prepare a machine without network access, run this once on a machine " + "with access and copy the cache folder: " + f'python -c "import tree_sitter_language_pack as t; t.download([{names}])"' + ) + self.grammar = grammar + + +@lru_cache(maxsize=None) +def get_pack_parser(grammar: str): + """Return the pack parser for ``grammar``, with a clear error when it is unavailable.""" + try: + return tslp.get_parser(grammar) + except tslp.DownloadError as e: + raise GrammarUnavailableError(grammar, e) from e + + +def ensure_pack_grammars(languages) -> None: + """Load every pack grammar that ``languages`` need, or raise ``GrammarUnavailableError``.""" + for language in sorted(set(languages)): + for grammar in PACK_GRAMMARS_BY_LANGUAGE.get(language, ()): + get_pack_parser(grammar) diff --git a/docker/Dockerfile b/docker/Dockerfile index d0a70d16..c6b9b086 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -18,6 +18,10 @@ COPY requirements.txt . # Install Python dependencies RUN pip install --no-cache-dir -r requirements.txt +# tree-sitter-language-pack 1.x downloads each grammar on first use. Fetch the +# ones the analyzers load at build time, so the image runs offline. +RUN python -c "import tree_sitter_language_pack as t; t.download(['dart', 'go', 'javascript', 'typescript', 'vue', 'zig'])" + # Copy application code COPY codewiki ./codewiki COPY img ./img diff --git a/guides/development.md b/guides/development.md index 4c88fb32..2c208019 100644 --- a/guides/development.md +++ b/guides/development.md @@ -122,7 +122,12 @@ Analyzers are standalone classes, one file per language. Follow 1. **Grammar.** Add the `tree-sitter-` package to `pyproject.toml` and `requirements.txt`, or take the grammar from `tree-sitter-language-pack` (already a dependency) with - `get_parser("")`, as `analyzers/dart.py` and `analyzers/go.py` do. + `get_pack_parser("")` from `utils/grammars.py`, as + `analyzers/dart.py` and `analyzers/go.py` do. Add the language and its + grammars to `PACK_GRAMMARS_BY_LANGUAGE` in the same file. The pack (1.x) downloads a grammar on first + use, so also add the name to the `t.download([...])` list in + `docker/Dockerfile`. `tests/test_pack_grammars.py` checks that the two + lists match. 2. **Analyzer.** Create `analyzers/.py` with a `TreeSitterAnalyzer` class that extracts components and relationships, plus a module-level `analyze__file(...)` function. diff --git a/pyproject.toml b/pyproject.toml index a407109a..a96d9216 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ dependencies = [ "GitPython>=3.1.40", "Jinja2>=3.1.6", "tree-sitter>=0.25", - "tree-sitter-language-pack>=0.8.0,<0.9.0", + "tree-sitter-language-pack>=1.23.1,<2.0.0", "tree-sitter-python>=0.23.6", "tree-sitter-java>=0.23.5", "tree-sitter-javascript>=0.21.4", diff --git a/requirements.txt b/requirements.txt index 7551bf8a..1d764c05 100644 --- a/requirements.txt +++ b/requirements.txt @@ -148,7 +148,7 @@ tree-sitter-embedded-template==0.23.2 tree-sitter-java==0.23.5 tree-sitter-javascript==0.21.4 tree-sitter-kotlin==1.1.0 -tree-sitter-language-pack==0.8.0 +tree-sitter-language-pack==1.23.1 tree-sitter-python==0.23.6 tree-sitter-ruby==0.23.1 tree-sitter-rust==0.23.2 diff --git a/tests/test_pack_grammars.py b/tests/test_pack_grammars.py new file mode 100644 index 00000000..88803fd9 --- /dev/null +++ b/tests/test_pack_grammars.py @@ -0,0 +1,80 @@ +"""Grammars from tree-sitter-language-pack: a failed download stops the +analysis once, with a clear message, and the Docker image fetches every +grammar the analyzers load.""" + +import re +from pathlib import Path + +import pytest +import tree_sitter_language_pack as tslp + +from codewiki.src.be.dependency_analyzer.analysis.call_graph_analyzer import ( + CallGraphAnalyzer, +) +from codewiki.src.be.dependency_analyzer.utils import grammars +from codewiki.src.be.dependency_analyzer.utils.grammars import ( + PACK_GRAMMARS, + GrammarUnavailableError, + ensure_pack_grammars, + get_pack_parser, +) + +DOCKERFILE = Path(__file__).resolve().parents[1] / "docker" / "Dockerfile" + + +@pytest.fixture +def offline(monkeypatch): + """Every pack grammar load fails as it does without network access.""" + calls = [] + + def fail(name): + calls.append(name) + raise tslp.DownloadError(f"Failed to fetch manifest for {name}: Connection refused") + + get_pack_parser.cache_clear() + monkeypatch.setattr(grammars.tslp, "get_parser", fail) + yield calls + get_pack_parser.cache_clear() + + +def test_download_failure_gives_a_clear_error(offline): + with pytest.raises(GrammarUnavailableError) as info: + get_pack_parser("go") + message = str(info.value) + assert info.value.grammar == "go" + assert "'go'" in message + assert "network access" in message + assert "Connection refused" in message + assert "t.download([" in message + assert isinstance(info.value.__cause__, tslp.DownloadError) + + +def test_only_the_grammars_of_present_languages_load(offline, monkeypatch): + monkeypatch.setattr(grammars.tslp, "get_parser", lambda name: offline.append(name)) + ensure_pack_grammars(["python", "vue", "java", "vue"]) + assert sorted(offline) == ["javascript", "typescript", "vue"] + + +def test_languages_without_pack_grammars_need_no_network(offline): + ensure_pack_grammars(["python", "java", "rust"]) + assert offline == [] + + +def test_analysis_stops_before_the_first_file(offline, tmp_path, monkeypatch): + (tmp_path / "main.go").write_text("package main\n\nfunc main() {}\n", encoding="utf-8") + analyzed = [] + monkeypatch.setattr( + CallGraphAnalyzer, "_analyze_code_file", lambda self, base, info: analyzed.append(info) + ) + with pytest.raises(GrammarUnavailableError, match="'go'"): + CallGraphAnalyzer().analyze_code_files( + [{"path": "main.go", "language": "go"}], str(tmp_path) + ) + assert analyzed == [] + + +def test_dockerfile_downloads_every_pack_grammar(): + match = re.search(r"t\.download\(\[([^\]]*)\]\)", DOCKERFILE.read_text(encoding="utf-8")) + assert match, "docker/Dockerfile must download the pack grammars at build time" + in_dockerfile = sorted(re.findall(r"'([^']+)'", match.group(1))) + assert in_dockerfile == list(PACK_GRAMMARS)