diff --git a/README.md b/README.md index 36a2235ba..52856e519 100644 --- a/README.md +++ b/README.md @@ -264,6 +264,7 @@ Codex users also need `multi_agent = true` under `[features]` in `~/.codex/confi | `sql` | SQL schema extraction | `uv tool install "graphifyy[sql]"` | | `postgres` | Live PostgreSQL introspection (`--postgres DSN`) | `uv tool install "graphifyy[postgres]"` | | `dm` | BYOND DreamMaker `.dm`/`.dme` AST extraction (may need a C compiler + `python3-dev` if no wheel matches your platform) | `uv tool install "graphifyy[dm]"` | +| `commonlisp` | Common Lisp `.lisp`/`.cl`/`.lsp`/`.asd` AST extraction | `uv tool install "graphifyy[commonlisp]"` | | `terraform` | Terraform / HCL `.tf`/`.tfvars`/`.hcl` AST extraction | `uv tool install "graphifyy[terraform]"` | | `pascal` | Pascal / Delphi `.pas`/`.dpr`/`.dpk`/`.inc` AST extraction (more accurate `calls`/`inherits` edges; falls back to a regex extractor when absent) | `uv tool install "graphifyy[pascal]"` | | `chinese` | Chinese query segmentation (jieba) | `uv tool install "graphifyy[chinese]"` | @@ -335,7 +336,7 @@ To remove graphify from all platforms at once: `graphify uninstall` (add `--purg | Type | Extensions | |------|-----------| -| Code (36 tree-sitter grammars) | `.py .ts .mts .cts .js .jsx .tsx .mjs .go .rs .java .c .cpp .cc .cxx .h .hpp .cu .cuh .metal .rb .cs .kt .kts .scala .php .swift .lua .luau .toc .zig .ps1 .psm1 .psd1 .ex .exs .m .mm .jl .vue .svelte .astro .groovy .gradle .dart .v .sv .svh .sql .f .f90 .f95 .f03 .f08 .pas .pp .dpr .dpk .lpr .inc .dfm .lfm .lpk .sh .bash .json .dm .dme .dmi .dmm .dmf .sln .slnx .csproj .fsproj .vbproj .xaml .razor .cshtml` (`.dm`/`.dme` requires `uv tool install graphifyy[dm]`; `.mts`/`.cts` reuse the TypeScript grammar, `.cc`/`.cxx` and CUDA `.cu`/`.cuh` and Metal `.metal` reuse the C++ grammar) | +| Code (37 tree-sitter grammars) | `.py .ts .mts .cts .js .jsx .tsx .mjs .go .rs .java .c .cpp .cc .cxx .h .hpp .cu .cuh .metal .rb .cs .kt .kts .scala .php .swift .lua .luau .toc .zig .ps1 .psm1 .psd1 .ex .exs .m .mm .jl .lisp .cl .lsp .asd .vue .svelte .astro .groovy .gradle .dart .v .sv .svh .sql .f .f90 .f95 .f03 .f08 .pas .pp .dpr .dpk .lpr .inc .dfm .lfm .lpk .sh .bash .json .dm .dme .dmi .dmm .dmf .sln .slnx .csproj .fsproj .vbproj .xaml .razor .cshtml` (`.dm`/`.dme` requires `uv tool install graphifyy[dm]`; `.lisp`/`.cl`/`.lsp`/`.asd` requires `uv tool install graphifyy[commonlisp]`; `.mts`/`.cts` reuse the TypeScript grammar, `.cc`/`.cxx` and CUDA `.cu`/`.cuh` and Metal `.metal` reuse the C++ grammar) | | Salesforce Apex | `.cls .trigger` (regex-based; classes, interfaces, enums, methods, triggers, SOQL/DML edges) | | Terraform / HCL | `.tf .tfvars .hcl` (requires `uv tool install graphifyy[terraform]`) | | MCP configs | `.mcp.json` `mcp.json` `mcp_servers.json` `claude_desktop_config.json` — extracts server nodes, package refs, env var requirements | diff --git a/graphify/detect.py b/graphify/detect.py index 0b569e4b7..7393805a6 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -28,7 +28,7 @@ class FileType(str, Enum): _MANIFEST_PATH = str(out_path("manifest.json")) -CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger'} +CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.jl', '.lisp', '.cl', '.lsp', '.asd', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger'} DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.skill', '.txt', '.rst', '.html', '.yaml', '.yml'} PAPER_EXTENSIONS = {'.pdf'} IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'} diff --git a/graphify/extract.py b/graphify/extract.py index 300a59618..f350e1c8f 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -34,6 +34,7 @@ from graphify.extractors.apex import extract_apex # noqa: F401 from graphify.extractors.bash import extract_bash # noqa: F401 from graphify.extractors.blade import extract_blade # noqa: F401 +from graphify.extractors.commonlisp import extract_commonlisp # noqa: F401 from graphify.extractors.csharp import ( CsharpNameResolver, _resolve_cross_file_csharp_imports, @@ -4094,6 +4095,10 @@ def add_existing_edge(edge: dict) -> None: ".sv": extract_verilog, ".svh": extract_verilog, ".sql": extract_sql, + ".lisp": extract_commonlisp, + ".cl": extract_commonlisp, + ".lsp": extract_commonlisp, + ".asd": extract_commonlisp, ".md": extract_markdown, ".mdx": extract_markdown, ".qmd": extract_markdown, diff --git a/graphify/extractors/__init__.py b/graphify/extractors/__init__.py index ada517094..68ff3340c 100644 --- a/graphify/extractors/__init__.py +++ b/graphify/extractors/__init__.py @@ -13,6 +13,7 @@ from graphify.extractors.apex import extract_apex from graphify.extractors.bash import extract_bash from graphify.extractors.blade import extract_blade +from graphify.extractors.commonlisp import extract_commonlisp from graphify.extractors.dart import extract_dart from graphify.extractors.dm import extract_dm, extract_dmf, extract_dmi, extract_dmm from graphify.extractors.elixir import extract_elixir @@ -37,6 +38,7 @@ "apex": extract_apex, "bash": extract_bash, "blade": extract_blade, + "commonlisp": extract_commonlisp, "dart": extract_dart, "delphi_form": extract_delphi_form, "dm": extract_dm, diff --git a/graphify/extractors/commonlisp.py b/graphify/extractors/commonlisp.py new file mode 100644 index 000000000..150880903 --- /dev/null +++ b/graphify/extractors/commonlisp.py @@ -0,0 +1,492 @@ +"""Common Lisp extractor for .lisp/.cl/.lsp/.asd, backed by tree-sitter-commonlisp.""" +from __future__ import annotations + +import warnings +from pathlib import Path + +from graphify.extractors.base import _make_id + + +# Standard CL definer forms that introduce data/type/variable bindings +# (not callable, no body to walk for calls) +_CL_DATA_DEFINERS = frozenset({ + "defstruct", "deftype", "define-condition", + "defvar", "defparameter", "defconstant", + "define-symbol-macro", +}) + +# Subset of data definers that take a VALUE as their second argument, which +# may itself be a string literal. For these, we must not mistake the value +# for a docstring. +_CL_VALUE_DEFINERS = frozenset({"defvar", "defparameter", "defconstant"}) + +# Standard CL macro-style definers +_CL_MACRO_DEFINERS = frozenset({ + "define-modify-macro", "define-compiler-macro", + "define-setf-expander", "defsetf", +}) + +# Symbols that start with "def" but are NOT definitions (denylist for the +# def-prefix heuristic that catches custom definers like definline-maybe) +_CL_NOT_DEFINERS = frozenset({ + "default", "default-value", "defaults", "define", + "defer", "deferred", "deflate", "deftest", +}) + +# CL special forms / macros that should not count as "calls" in the graph +_CL_SPECIAL_FORMS = frozenset({ + "let", "let*", "flet", "labels", "macrolet", + "if", "when", "unless", "cond", "case", "ecase", "typecase", "etypecase", + "progn", "prog1", "prog2", "block", "return-from", "tagbody", "go", + "lambda", "function", "funcall", "apply", + "setf", "setq", "psetf", "psetq", "incf", "decf", "push", "pop", + "loop", "do", "do*", "dolist", "dotimes", "map", "mapcar", "mapc", + "format", "print", "princ", "prin1", "write", "terpri", + "and", "or", "not", "null", + "values", "multiple-value-bind", "multiple-value-setq", + "declare", "the", "locally", "eval-when", + "quote", "backquote", + "error", "signal", "warn", "assert", "check-type", + "handler-case", "handler-bind", "restart-case", "ignore-errors", + "unwind-protect", "catch", "throw", + "with-open-file", "with-output-to-string", "with-input-from-string", + "with-slots", "with-accessors", + "defvar", "defparameter", "defconstant", + "in-package", "require", "provide", "use-package", + "t", "nil", +}) + + +def extract_commonlisp(path: Path) -> dict: + """Extract packages, classes, functions, methods, macros, and calls from a Common Lisp file.""" + try: + import tree_sitter_commonlisp as tscl + from tree_sitter import Language, Parser + except ImportError: + return {"nodes": [], "edges": [], "error": "tree-sitter-commonlisp not installed"} + + try: + # tree-sitter-commonlisp 0.4.1 (latest) ships the old binding that returns + # an int pointer from language(); tree_sitter.Language only accepts that + # int via a deprecated path. Silence that one warning at the call site + # until the grammar ships a PyCapsule binding. + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message="int argument support is deprecated", + category=DeprecationWarning, + ) + language = Language(tscl.language()) + parser = Parser(language) + source = path.read_bytes() + tree = parser.parse(source) + root = tree.root_node + except Exception as e: + return {"nodes": [], "edges": [], "error": str(e)} + + stem = path.stem + str_path = str(path) + nodes: list[dict] = [] + edges: list[dict] = [] + seen_ids: set[str] = set() + function_bodies: list[tuple[str, object]] = [] # (nid, body_nodes) + current_package: str | None = None + + def _text(node) -> str: + return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace") + + # CL symbols can contain operator chars like = < > ? ! + * / that the + # generic _make_id strips. Map them to readable suffixes so upi=, upi<, + # upi> become distinct ids. + _CL_CHAR_MAP = { + '=': '_eq', '<': '_lt', '>': '_gt', '?': '_p', '!': '_bang', + '+': '_plus', '*': '_star', '/': '_slash', '%': '_pct', '&': '_amp', + } + + def _cl_id(*parts: str) -> str: + normalized = [] + for p in parts: + normalized.append(''.join(_CL_CHAR_MAP.get(c, c) for c in p)) + return _make_id(*normalized) + + def add_node(nid: str, label: str, line: int, file_type: str = "code") -> None: + if nid not in seen_ids: + seen_ids.add(nid) + nodes.append({ + "id": nid, + "label": label, + "file_type": file_type, + "source_file": str_path, + "source_location": f"L{line}", + }) + + def add_edge(src: str, tgt: str, relation: str, line: int, + confidence: str = "EXTRACTED", weight: float = 1.0) -> None: + edges.append({ + "source": src, + "target": tgt, + "relation": relation, + "confidence": confidence, + "source_file": str_path, + "source_location": f"L{line}", + "weight": weight, + }) + + file_nid = _cl_id(stem) + add_node(file_nid, path.name, 1) + + def _first_sym(node) -> str | None: + """Get the first sym_lit text from a list_lit's children.""" + for child in node.children: + if child.type == "sym_lit": + return _text(child) + return None + + def _kwd_text(node) -> str: + """Extract the symbol name from a kwd_lit, stripping the leading colon.""" + t = _text(node) + return t.lstrip(":").lstrip("#:") + + def _handle_defpackage(node) -> None: + nonlocal current_package + children = node.children + pkg_name = None + for child in children: + if child.type == "kwd_lit" and pkg_name is None: + pkg_name = _kwd_text(child) + break + if child.type == "sym_lit" and _text(child) != "defpackage": + pkg_name = _text(child) + break + if not pkg_name: + return + current_package = pkg_name + pkg_nid = _cl_id(stem, pkg_name) + line = node.start_point[0] + 1 + add_node(pkg_nid, pkg_name, line) + add_edge(file_nid, pkg_nid, "contains", line) + + # Extract :use clauses as imports + for child in children: + if child.type == "list_lit": + first = _first_sym(child) + if first is None: + # Could be (:use ...) with kwd_lit + for gc in child.children: + if gc.type == "kwd_lit" and _kwd_text(gc) == "use": + for uc in child.children: + if uc.type == "kwd_lit" and uc != gc: + mod_name = _kwd_text(uc) + if mod_name != "use": + tgt_nid = _cl_id(mod_name) + add_edge(pkg_nid, tgt_nid, "imports", + child.start_point[0] + 1) + break + + def _handle_defclass(node) -> None: + children = node.children + # Find class name: second sym_lit after "defclass" + sym_lits = [c for c in children if c.type == "sym_lit"] + if len(sym_lits) < 2: + return + class_name = _text(sym_lits[1]) + line = node.start_point[0] + 1 + class_nid = _cl_id(stem, class_name) + add_node(class_nid, class_name, line) + + parent_nid = file_nid + if current_package: + pkg_nid = _cl_id(stem, current_package) + if pkg_nid in seen_ids: + parent_nid = pkg_nid + add_edge(parent_nid, class_nid, "contains", line) + + # Superclasses (third element, a list) + list_lits = [c for c in children if c.type == "list_lit"] + if list_lits: + superclass_list = list_lits[0] + for sc in superclass_list.children: + if sc.type == "sym_lit": + sc_name = _text(sc) + sc_nid = _cl_id(stem, sc_name) + add_edge(class_nid, sc_nid, "inherits", + superclass_list.start_point[0] + 1) + + def _handle_defun_node(node) -> None: + """Handle a defun AST node (covers defun, defmethod, defgeneric, defmacro).""" + header = None + for child in node.children: + if child.type == "defun_header": + header = child + break + if not header: + return + + # Determine keyword type + keyword_type = "defun" + func_name = None + for child in header.children: + if child.type == "defun_keyword": + for kc in child.children: + if kc.type in ("defun", "defmethod", "defgeneric", "defmacro"): + keyword_type = kc.type + break + if child.type == "sym_lit" and func_name is None: + func_name = _text(child) + + if not func_name: + return + + line = node.start_point[0] + 1 + func_nid = _cl_id(stem, func_name) + + if keyword_type == "defmethod": + label = f".{func_name}()" + elif keyword_type == "defmacro": + label = f"{func_name} (macro)" + elif keyword_type == "defgeneric": + label = f"{func_name} (generic)" + else: + label = f"{func_name}()" + + add_node(func_nid, label, line) + + parent_nid = file_nid + if current_package: + pkg_nid = _cl_id(stem, current_package) + if pkg_nid in seen_ids: + parent_nid = pkg_nid + if keyword_type == "defmethod": + add_edge(parent_nid, func_nid, "method", line) + else: + add_edge(parent_nid, func_nid, "contains", line) + + # Method specializers: (defmethod name ((param class) ...)) + if keyword_type == "defmethod": + for child in header.children: + if child.type == "list_lit": + # This is the parameter list + for param in child.children: + if param.type == "list_lit": + syms = [c for c in param.children if c.type == "sym_lit"] + if len(syms) >= 2: + specializer_name = _text(syms[1]) + spec_nid = _cl_id(stem, specializer_name) + add_edge(func_nid, spec_nid, "specializes", + param.start_point[0] + 1) + break + + # Docstring + for child in node.children: + if child.type == "str_lit": + doc_text = _text(child).strip('"') + if doc_text: + doc_nid = _cl_id(func_nid, "rationale") + add_node(doc_nid, doc_text[:120], child.start_point[0] + 1, file_type="rationale") + add_edge(doc_nid, func_nid, "rationale_for", + child.start_point[0] + 1) + break + + # Collect body for call extraction + body_nodes = [c for c in node.children + if c.type not in ("(", ")", "defun_header", "str_lit")] + if body_nodes: + function_bodies.append((func_nid, body_nodes)) + + def _extract_def_name(node, def_keyword: str) -> str | None: + """Get the name being defined. May be a sym_lit or a list_lit whose + first sym_lit is the name (e.g. (defstruct (foo :conc-name "BAR-") ...)).""" + seen_keyword = False + for child in node.children: + if not seen_keyword: + if child.type == "sym_lit" and _text(child).lower() == def_keyword: + seen_keyword = True + continue + if child.type == "sym_lit": + return _text(child) + if child.type == "list_lit": + for gc in child.children: + if gc.type == "sym_lit": + return _text(gc) + return None + return None + + def _collect_def_body(node, def_keyword: str) -> list: + """Collect body forms from (DEFKEYWORD name (params) [doc] body...). + Skips the def keyword, the name, the params list (if present), and + a leading docstring str_lit.""" + children = [c for c in node.children if c.type not in ("(", ")")] + idx = 0 + # Skip def keyword + if idx < len(children) and children[idx].type == "sym_lit" \ + and _text(children[idx]).lower() == def_keyword: + idx += 1 + # Skip name (sym_lit or list_lit) + if idx < len(children) and children[idx].type in ("sym_lit", "list_lit"): + idx += 1 + # Skip params list (the next list_lit, if any) + if idx < len(children) and children[idx].type == "list_lit": + idx += 1 + # Skip leading docstring + if idx < len(children) and children[idx].type == "str_lit": + idx += 1 + return children[idx:] + + def _handle_def_form(node, def_keyword: str) -> None: + """Handle a generic (DEFKEYWORD name ...) form. Covers standard CL + definers (defstruct, deftype, defvar, define-condition, etc.) and + custom def-prefixed macros (definline, definline-maybe, etc.).""" + name = _extract_def_name(node, def_keyword) + if not name: + return + line = node.start_point[0] + 1 + nid = _cl_id(stem, name) + + kw = def_keyword.lower() + if kw in _CL_DATA_DEFINERS: + label = name + is_callable = False + elif kw in _CL_MACRO_DEFINERS or "macro" in kw: + label = f"{name} (macro)" + is_callable = True + else: + # Custom def-prefixed: assume function-like + label = f"{name}()" + is_callable = True + + add_node(nid, label, line) + + parent_nid = file_nid + if current_package: + pkg_nid = _cl_id(stem, current_package) + if pkg_nid in seen_ids: + parent_nid = pkg_nid + add_edge(parent_nid, nid, "contains", line) + + # Docstring. Skip for defvar/defparameter/defconstant — their second + # argument is a value (possibly a string literal) which would be + # wrongly captured as a docstring. + if kw not in _CL_VALUE_DEFINERS: + for child in node.children: + if child.type == "str_lit": + doc_text = _text(child).strip('"') + if doc_text: + doc_nid = _cl_id(nid, "rationale") + add_node(doc_nid, doc_text[:120], child.start_point[0] + 1, file_type="rationale") + add_edge(doc_nid, nid, "rationale_for", + child.start_point[0] + 1) + break + + if is_callable: + body_nodes = _collect_def_body(node, def_keyword) + if body_nodes: + function_bodies.append((nid, body_nodes)) + + def _is_def_prefixed(sym: str) -> bool: + """Heuristic: does this symbol look like a definition macro?""" + if sym in _CL_NOT_DEFINERS: + return False + return sym.startswith("def") or sym.startswith("define-") + + def _process_form(top) -> bool: + """Dispatch on a single list_lit form. Returns True if it was + recognized as a definition or package directive (caller must NOT + recurse into it). Returns False if unrecognized, so the caller can + recurse to find defs nested inside wrapper macros like + (optimizing ...), (eval-when ...), (progn ...), etc.""" + nonlocal current_package + + # Check for defun node type inside list_lit + for child in top.children: + if child.type == "defun": + _handle_defun_node(child) + return True + + first = _first_sym(top) + if not first: + return False + first_lower = first.lower() + + if first_lower == "defpackage": + _handle_defpackage(top) + return True + if first_lower == "in-package": + for child in top.children: + if child.type == "kwd_lit": + current_package = _kwd_text(child) + break + if child.type == "sym_lit" and _text(child) != "in-package": + current_package = _text(child) + break + return True + if first_lower == "defclass": + _handle_defclass(top) + return True + if first_lower in ("require", "ql:quickload"): + for child in top.children: + if child.type in ("kwd_lit", "str_lit"): + mod_name = _kwd_text(child) if child.type == "kwd_lit" else _text(child).strip('"') + if mod_name: + tgt_nid = _cl_id(mod_name) + add_edge(file_nid, tgt_nid, "imports", + top.start_point[0] + 1) + break + return True + if first_lower in _CL_DATA_DEFINERS or first_lower in _CL_MACRO_DEFINERS: + _handle_def_form(top, first_lower) + return True + if _is_def_prefixed(first_lower): + # Custom definer (definline, definline-maybe, defcomponent, etc.) + _handle_def_form(top, first_lower) + return True + + return False + + def _walk_forms(parent) -> None: + """Walk list_lit children of `parent`, processing each. If a form + isn't recognized, recurse into it — many CL codebases wrap + definitions in macros like (optimizing ...), (eval-when ...), or + (progn ...) that aren't themselves definers but contain defs. + Also descends into reader conditionals (#+feature / #-feature), + which wrap their guarded form in an include_reader_macro node.""" + for top in parent.children: + if top.type == "list_lit": + if not _process_form(top): + _walk_forms(top) + elif top.type == "include_reader_macro": + _walk_forms(top) + + _walk_forms(root) + + # Call extraction pass + label_to_nid: dict[str, str] = {} + for n in nodes: + raw = n["label"] + normalised = raw.replace(" (macro)", "").replace(" (generic)", "").strip("()").lstrip(".") + label_to_nid[normalised.lower()] = n["id"] + + seen_call_pairs: set[tuple[str, str]] = set() + + def walk_calls(node, caller_nid: str) -> None: + if node.type == "defun": + return + if node.type == "list_lit": + callee = _first_sym(node) + if callee and callee.lower() not in _CL_SPECIAL_FORMS: + tgt_nid = label_to_nid.get(callee.lower()) + if tgt_nid and tgt_nid != caller_nid: + pair = (caller_nid, tgt_nid) + if pair not in seen_call_pairs: + seen_call_pairs.add(pair) + add_edge(caller_nid, tgt_nid, "calls", + node.start_point[0] + 1, confidence="EXTRACTED", weight=1.0) + for child in node.children: + walk_calls(child, caller_nid) + + for caller_nid, body_nodes in function_bodies: + for body_node in body_nodes: + walk_calls(body_node, caller_nid) + + clean_edges = [e for e in edges if e["source"] in seen_ids and + (e["target"] in seen_ids or e["relation"] in ("imports", "imports_from"))] + return {"nodes": nodes, "edges": clean_edges} diff --git a/pyproject.toml b/pyproject.toml index 619f00af5..98d5d600b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,12 +80,13 @@ sql = ["tree-sitter-sql"] # absent (#781), so this stays optional. Unlike tree-sitter-dm below, it ships # prebuilt wheels for every platform (win/macOS/Linux), so no C toolchain needed. pascal = ["tree-sitter-pascal"] +commonlisp = ["tree-sitter-commonlisp"] # tree-sitter-dm (BYOND DreamMaker) ships only a Windows wheel, so on Linux/Mac it # must compile from source (needs a C toolchain + python3-dev). Keeping it optional # avoids breaking the default `uv tool install graphifyy` for everyone (#1104). dm = ["tree-sitter-dm"] terraform = ["tree-sitter-hcl"] -all = ["mcp>=1,<3", "starlette>=1.3.1,<2", "neo4j", "falkordb", "pypdf>=6.12.0", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper; python_version >= '3.11'", "yt-dlp>=2026.6.9", "matplotlib", "numpy>=2.0; python_version >= '3.13'", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba", "tree-sitter-dm", "tree-sitter-hcl", "tree-sitter-pascal"] +all = ["mcp>=1,<3", "starlette>=1.3.1,<2", "neo4j", "falkordb", "pypdf>=6.12.0", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper; python_version >= '3.11'", "yt-dlp>=2026.6.9", "matplotlib", "numpy>=2.0; python_version >= '3.13'", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba", "tree-sitter-dm", "tree-sitter-hcl", "tree-sitter-pascal", "tree-sitter-commonlisp"] [project.scripts] graphify = "graphify.__main__:main" diff --git a/tests/fixtures/sample.lisp b/tests/fixtures/sample.lisp new file mode 100644 index 000000000..4dc2dc417 --- /dev/null +++ b/tests/fixtures/sample.lisp @@ -0,0 +1,70 @@ +(defpackage :http-server + (:use :cl :alexandria) + (:export #:make-server #:start #:stop)) + +(in-package :http-server) + +(deftype port-number () '(integer 1 65535)) + +(defstruct request-stats + bytes-in + bytes-out + duration-ms) + +(defstruct (connection (:conc-name conn-)) + id + socket + state) + +(defvar *active-connections* nil) +(defparameter *default-port* 8080) +(defconstant +max-headers+ 100) + +(define-condition server-error (error) + ((reason :initarg :reason :reader error-reason))) + +(defclass server () + ((host :initarg :host :accessor server-host) + (port :initarg :port :accessor server-port) + (handler :initarg :handler :accessor server-handler))) + +(defclass ssl-server (server) + ((cert-path :initarg :cert-path :accessor ssl-cert-path))) + +(defgeneric process-request (server request)) + +(defmethod process-request ((srv server) (req string)) + "Process an incoming HTTP request." + (let ((parsed (parse-headers req))) + (funcall (server-handler srv) parsed))) + +;; Custom definer (Franz-style) — should be picked up by the def-prefix heuristic +(definline-maybe header= (a b) + "Fast header equality." + (string-equal a b)) + +(definline header< (a b) + (string< a b)) + +(defun make-server (host port handler) + "Create a new server instance." + (make-instance 'server :host host :port port :handler handler)) + +(defun start (server) + "Start the server listening on its configured port." + (format t "Starting server on ~a:~a~%" (server-host server) (server-port server)) + (process-request server "GET / HTTP/1.1")) + +(defun stop (server) + (format t "Stopping server~%")) + +(defun compare-headers (h1 h2) + "Compare two headers using the custom definers." + (or (header= h1 h2) (header< h1 h2))) + +(defmacro with-server ((var host port handler) &body body) + "Execute body with a running server bound to var." + `(let ((,var (make-server ,host ,port ,handler))) + (unwind-protect + (progn (start ,var) ,@body) + (stop ,var)))) diff --git a/tests/test_languages.py b/tests/test_languages.py index aff689d2a..d2e4005b3 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -8,7 +8,7 @@ extract_swift, extract_go, extract_julia, extract_js, extract_fortran, extract_groovy, extract_sln, extract_csproj, extract_xaml, extract_razor, extract_dm, extract_dmi, extract_dmm, extract_dmf, - extract_powershell, extract_apex, extract_verilog, + extract_powershell, extract_apex, extract_commonlisp, extract_verilog, extract_powershell_manifest, ) @@ -22,6 +22,10 @@ _ilu.find_spec("tree_sitter_dm") is None, reason="tree-sitter-dm not installed (optional [dm] extra)", ) +_needs_commonlisp = pytest.mark.skipif( + _ilu.find_spec("tree_sitter_commonlisp") is None, + reason="tree-sitter-commonlisp not installed (optional [commonlisp] extra)", +) def _labels(r): @@ -2999,3 +3003,238 @@ def test_decldef_merge_does_not_merge_same_name_same_dir_distinct_files(): r = _corpus("cpp_samedir/Alpha.h", "cpp_samedir/Beta.h") dups = _nodes_with_label(r, "Dup") assert len(dups) == 2, f"same-dir distinct Dups must stay distinct, got {[n['id'] for n in dups]}" + +# ── Common Lisp ────────────────────────────────────────────────────────────── + +@_needs_commonlisp +def test_cl_finds_package(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert "error" not in r + assert "http-server" in _labels(r) + +@_needs_commonlisp +def test_cl_finds_class(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert "server" in _labels(r) + assert "ssl-server" in _labels(r) + +@_needs_commonlisp +def test_cl_finds_defun(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + labels = _labels(r) + assert any("make-server" in l for l in labels) + assert any("start" in l for l in labels) + assert any("stop" in l for l in labels) + +@_needs_commonlisp +def test_cl_finds_generic(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert any("process-request" in l for l in _labels(r)) + +@_needs_commonlisp +def test_cl_finds_macro(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert any("with-server" in l and "macro" in l for l in _labels(r)) + +@_needs_commonlisp +def test_cl_emits_calls(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + calls = _calls(r) + # start() calls process-request + assert any("start" in src and "process-request" in tgt for src, tgt in calls) + # with-server macro calls make-server and start + assert any("with-server" in src and "make-server" in tgt for src, tgt in calls) + assert any("with-server" in src and "start" in tgt for src, tgt in calls) + +@_needs_commonlisp +def test_cl_calls_are_extracted(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + for e in r["edges"]: + if e["relation"] == "calls": + assert e["confidence"] == "EXTRACTED" + +@_needs_commonlisp +def test_cl_no_dangling_edges(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + node_ids = {n["id"] for n in r["nodes"]} + for e in r["edges"]: + if e["relation"] in ("contains", "method", "calls"): + assert e["source"] in node_ids + +@_needs_commonlisp +def test_cl_docstrings(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + rationale_edges = [e for e in r["edges"] if e["relation"] == "rationale_for"] + assert len(rationale_edges) >= 3 # make-server, start, process-request have docstrings + labels = _labels(r) + assert any("Process an incoming" in l for l in labels) + +@_needs_commonlisp +def test_cl_method_specializers(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + spec_edges = [e for e in r["edges"] if e["relation"] == "specializes"] + assert len(spec_edges) >= 1 + # process-request specializes on server + assert any("process_request" in e["source"] and "server" in e["target"] for e in spec_edges) + +@_needs_commonlisp +def test_cl_inherits(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + inherit_edges = [e for e in r["edges"] if e["relation"] == "inherits"] + assert len(inherit_edges) >= 1 + assert any("ssl_server" in e["source"] and "server" in e["target"] for e in inherit_edges) + +@_needs_commonlisp +def test_cl_imports(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + import_edges = [e for e in r["edges"] if e["relation"] == "imports"] + targets = {e["target"] for e in import_edges} + assert "cl" in targets + assert "alexandria" in targets + +@_needs_commonlisp +def test_cl_finds_deftype(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert "port-number" in _labels(r) + +@_needs_commonlisp +def test_cl_finds_defstruct(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + labels = _labels(r) + assert "request-stats" in labels + # defstruct with options form: (defstruct (name ...) ...) + assert "connection" in labels + +@_needs_commonlisp +def test_cl_finds_defvar_defparameter_defconstant(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + labels = _labels(r) + assert "*active-connections*" in labels + assert "*default-port*" in labels + assert "+max-headers+" in labels + +@_needs_commonlisp +def test_cl_finds_define_condition(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert "server-error" in _labels(r) + +@_needs_commonlisp +def test_cl_finds_custom_definer(): + """The def-prefix heuristic should catch definline / definline-maybe.""" + r = extract_commonlisp(FIXTURES / "sample.lisp") + labels = _labels(r) + # definline-maybe and definline should produce function-style nodes + assert "header=()" in labels + assert "header<()" in labels + +@_needs_commonlisp +def test_cl_custom_definer_in_call_graph(): + """Functions defined via custom definers should appear in the call graph.""" + r = extract_commonlisp(FIXTURES / "sample.lisp") + calls = _calls(r) + # compare-headers calls header= and header< (defined via definline-maybe / definline) + assert any("compare-headers" in src and "header=" in tgt for src, tgt in calls) + assert any("compare-headers" in src and "header<" in tgt for src, tgt in calls) + +@_needs_commonlisp +def test_cl_operator_names_disambiguated(): + """upi=, upi<, upi> must produce distinct ids (operator chars matter).""" + r = extract_commonlisp(FIXTURES / "sample.lisp") + # header= and header< must have different ids + eq_ids = [n["id"] for n in r["nodes"] if n["label"] == "header=()"] + lt_ids = [n["id"] for n in r["nodes"] if n["label"] == "header<()"] + assert len(eq_ids) == 1 + assert len(lt_ids) == 1 + assert eq_ids[0] != lt_ids[0] + +@_needs_commonlisp +def test_cl_default_value_not_treated_as_definition(): + """The def-prefix heuristic must not match denylisted symbols.""" + import tempfile + code = "(in-package :cl-user)\n(default-value foo)\n" + with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: + f.write(code) + path = Path(f.name) + try: + r = extract_commonlisp(path) + labels = _labels(r) + # default-value is denylisted, shouldn't create a "foo" node + assert "foo" not in labels + assert "foo()" not in labels + finally: + path.unlink() + +@_needs_commonlisp +def test_cl_defs_inside_wrapper_macro(): + """Definitions nested inside wrapper macros like (optimizing ...) or + (eval-when ...) must be extracted. Many CL codebases wrap hot-path + inline functions in application-specific macros.""" + import tempfile + code = """(in-package :cl-user) +(optimizing + (definline-maybe packet-type (p) (aref p 0)) + (definline-maybe set-packet-type (p v) (setf (aref p 0) v))) +(eval-when (:compile-toplevel :load-toplevel :execute) + (defun helper () 42)) +(progn + (defun progn-def () 'ok)) +""" + with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: + f.write(code) + path = Path(f.name) + try: + r = extract_commonlisp(path) + labels = _labels(r) + assert "packet-type()" in labels + assert "set-packet-type()" in labels + assert "helper()" in labels + assert "progn-def()" in labels + finally: + path.unlink() + +@_needs_commonlisp +def test_cl_defs_inside_reader_conditional(): + """#+feature / #-feature reader conditionals wrap their guarded form + in an include_reader_macro AST node, which the walker must descend + into to find the nested definition.""" + import tempfile + code = """(in-package :cl-user) +#+little-endian +(definline-maybe byte-hash= (a b) (eq a b)) +#-sbcl +(defun only-on-non-sbcl () 1) +""" + with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: + f.write(code) + path = Path(f.name) + try: + r = extract_commonlisp(path) + labels = _labels(r) + assert "byte-hash=()" in labels + assert "only-on-non-sbcl()" in labels + finally: + path.unlink() + +@_needs_commonlisp +def test_cl_defparameter_string_value_not_docstring(): + """For defvar/defparameter/defconstant, a string literal in the VALUE + position must not be wrongly captured as a docstring node.""" + import tempfile + code = '''(in-package :cl-user) +(defparameter *config-path* "/etc/app/config") +(defvar *greeting* "hello world") +''' + with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: + f.write(code) + path = Path(f.name) + try: + r = extract_commonlisp(path) + labels = _labels(r) + assert "*config-path*" in labels + assert "*greeting*" in labels + # The string VALUES must not show up as rationale nodes + assert not any("/etc/app/config" in l for l in labels) + assert not any("hello world" in l for l in labels) + finally: + path.unlink() +