diff --git a/.doctrine/project.json b/.doctrine/project.json index ae25377..899af3f 100644 --- a/.doctrine/project.json +++ b/.doctrine/project.json @@ -1,155 +1,155 @@ { - "schemaVersion": 1, - "project": { - "repo": "SylphxAI/coderag", - "name": "SylphxAI CodeRAG", - "lifecycle": "active", - "layer": "tooling", - "policyPool": "agent-tool", - "summary": "TypeScript/Bun monorepo for hybrid code search and an MCP server for AI assistant code retrieval.", - "goals": [ - "Provide the @sylphx/coderag core package for hybrid TF-IDF, vector, AST chunking, indexing, and persistent search.", - "Provide the @sylphx/coderag-mcp package as an MCP server surface for AI assistants and IDEs.", - "Maintain docs, examples, tests, and package release workflows for the CodeRAG ecosystem." - ], - "nonGoals": [ - "Own IDE products, AI assistant runtime policy, or downstream coding-agent behavior.", - "Own embedding provider accounts, external vector services, or customer-specific codebase indexes.", - "Own enterprise engineering doctrine, standards, or project manifest schema." - ] - }, - "boundaries": { - "owns": [ - { - "name": "coderag-core", - "description": "The core indexing, TF-IDF/vector search, AST chunking, embeddings, storage, cache, and language configuration packages under packages/core." - }, - { - "name": "coderag-mcp-server", - "description": "The MCP server package, tool surface, installation docs, configuration docs, and integration examples under packages/mcp-server and docs/mcp." - } - ], - "doesNotOwn": [ - "IDE products, assistant UX, model routing policy, or coding-agent runtime behavior.", - "Embedding provider accounts, external vector databases, or customer-specific codebase indexes.", - "Enterprise doctrine and manifest schema owned by SylphxAI/doctrine." - ], - "publicSurfaces": [ - { - "type": "documentation", - "name": "Repository README", - "location": "README.md" - }, - { - "type": "manifest", - "name": "Root package manifest and scripts", - "location": "package.json" - }, - { - "type": "package", - "name": "@sylphx/coderag", - "location": "packages/core/" - }, - { - "type": "package", - "name": "@sylphx/coderag-mcp", - "location": "packages/mcp-server/" - }, - { - "type": "documentation", - "name": "Documentation site", - "location": "docs/" - }, - { - "type": "documentation", - "name": "MCP integration docs", - "location": "docs/mcp/" - }, - { - "type": "test", - "name": "Package tests", - "location": "packages/*/src/*.test.ts" - }, - { - "type": "workflow", - "name": "CI and release workflows", - "location": ".github/workflows/" - } - ], - "allowedDependencies": [ - { - "repo": "SylphxAI/doctrine", - "surface": "project manifest schema and engineering doctrine", - "direction": "peer-public" - }, - { - "repo": "SylphxAI/.github", - "surface": "reusable release workflow", - "direction": "upstream-shared" - }, - { - "repo": "SylphxAI/synth", - "surface": "parser packages for AST chunking", - "direction": "upstream-shared" - } - ], - "forbiddenCouplings": [ - "Do not encode a specific IDE, assistant, model provider, or customer codebase policy into the core search engine.", - "Do not store or assume embedding provider credentials in package code or docs examples.", - "Do not fork doctrine standards into this repository." - ] - }, - "documentation": { - "adr": { - "path": "docs/adr/", - "status": "planned" - }, - "specs": { - "path": "docs/guide/", - "status": "present" - }, - "catalog": { - "path": ".doctrine/project.json", - "status": "present" - }, - "runbooks": { - "path": "docs/mcp/", - "status": "present" - }, - "generatedReferences": { - "path": "docs/api/", - "status": "present" - } - }, - "delivery": { - "ciModel": "legacy-ci", - "requiredContexts": [], - "deployPath": "Pull requests and merge queue run Bun/Turborepo CI; main-branch Release delegates to SylphxAI/.github reusable release workflow for package publication.", - "productionProof": "Passing typecheck, tests, package build, MCP smoke/config validation when MCP behavior changes, successful release workflow, and package-registry/docs readback for published versions.", - "recoveryClass": "forward-fix-only", - "deployable": false - }, - "adoption": { - "status": "baseline", - "gaps": [ - { - "id": "required-contexts-not-recorded", - "description": "CI exists for pull requests and merge queue, but required branch protection contexts are not recorded in this manifest baseline.", - "owner": "SylphxAI/coderag", - "target": "central ADR-29 required-context rollout or repo-local branch protection audit" - }, - { - "id": "embedding-provider-boundary", - "description": "Optional vector search can involve provider credentials or external services; keep provider configuration outside package internals and validate through MCP/config tests.", - "owner": "SylphxAI/coderag", - "target": "next embedding-provider or MCP config change" - }, - { - "id": "package-release-forward-fix", - "description": "Published package versions and MCP behavior are forward-only for consumers; add release recovery/runbook proof before production lifecycle promotion.", - "owner": "SylphxAI/coderag", - "target": "before production lifecycle promotion" - } - ] - } + "schemaVersion": 1, + "project": { + "repo": "SylphxAI/coderag", + "name": "SylphxAI CodeRAG", + "lifecycle": "active", + "layer": "tooling", + "policyPool": "agent-tool", + "summary": "TypeScript/Bun monorepo for hybrid code search and an MCP server for AI assistant code retrieval.", + "goals": [ + "Provide the @sylphx/coderag core package for hybrid TF-IDF, vector, AST chunking, indexing, and persistent search.", + "Provide the @sylphx/coderag-mcp package as an MCP server surface for AI assistants and IDEs.", + "Maintain docs, examples, tests, and package release workflows for the CodeRAG ecosystem." + ], + "nonGoals": [ + "Own IDE products, AI assistant runtime policy, or downstream coding-agent behavior.", + "Own embedding provider accounts, external vector services, or customer-specific codebase indexes.", + "Own enterprise engineering doctrine, standards, or project manifest schema." + ] + }, + "boundaries": { + "owns": [ + { + "name": "coderag-core", + "description": "The core indexing, TF-IDF/vector search, AST chunking, embeddings, storage, cache, and language configuration packages under packages/core." + }, + { + "name": "coderag-mcp-server", + "description": "The MCP server package, tool surface, installation docs, configuration docs, and integration examples under packages/mcp-server and docs/mcp." + } + ], + "doesNotOwn": [ + "IDE products, assistant UX, model routing policy, or coding-agent runtime behavior.", + "Embedding provider accounts, external vector databases, or customer-specific codebase indexes.", + "Enterprise doctrine and manifest schema owned by SylphxAI/doctrine." + ], + "publicSurfaces": [ + { + "type": "documentation", + "name": "Repository README", + "location": "README.md" + }, + { + "type": "manifest", + "name": "Root package manifest and scripts", + "location": "package.json" + }, + { + "type": "package", + "name": "@sylphx/coderag", + "location": "packages/core/" + }, + { + "type": "package", + "name": "@sylphx/coderag-mcp", + "location": "packages/mcp-server/" + }, + { + "type": "documentation", + "name": "Documentation site", + "location": "docs/" + }, + { + "type": "documentation", + "name": "MCP integration docs", + "location": "docs/mcp/" + }, + { + "type": "test", + "name": "Package tests", + "location": "packages/*/src/*.test.ts" + }, + { + "type": "workflow", + "name": "CI and release workflows", + "location": ".github/workflows/" + } + ], + "allowedDependencies": [ + { + "repo": "SylphxAI/doctrine", + "surface": "project manifest schema and engineering doctrine", + "direction": "peer-public" + }, + { + "repo": "SylphxAI/.github", + "surface": "reusable release workflow", + "direction": "upstream-shared" + }, + { + "repo": "SylphxAI/synth", + "surface": "parser packages for AST chunking", + "direction": "upstream-shared" + } + ], + "forbiddenCouplings": [ + "Do not encode a specific IDE, assistant, model provider, or customer codebase policy into the core search engine.", + "Do not store or assume embedding provider credentials in package code or docs examples.", + "Do not fork doctrine standards into this repository." + ] + }, + "documentation": { + "adr": { + "path": "docs/adr/", + "status": "planned" + }, + "specs": { + "path": "docs/guide/", + "status": "present" + }, + "catalog": { + "path": ".doctrine/project.json", + "status": "present" + }, + "runbooks": { + "path": "docs/mcp/", + "status": "present" + }, + "generatedReferences": { + "path": "docs/api/", + "status": "present" + } + }, + "delivery": { + "ciModel": "legacy-ci", + "requiredContexts": [], + "deployPath": "Pull requests and merge queue run Bun/Turborepo CI; main-branch Release delegates to SylphxAI/.github reusable release workflow for package publication.", + "productionProof": "Passing typecheck, tests, package build, MCP smoke/config validation when MCP behavior changes, successful release workflow, and package-registry/docs readback for published versions.", + "recoveryClass": "forward-fix-only", + "deployable": false + }, + "adoption": { + "status": "baseline", + "gaps": [ + { + "id": "required-contexts-not-recorded", + "description": "CI exists for pull requests and merge queue, but required branch protection contexts are not recorded in this manifest baseline.", + "owner": "SylphxAI/coderag", + "target": "central ADR-29 required-context rollout or repo-local branch protection audit" + }, + { + "id": "embedding-provider-boundary", + "description": "Optional vector search can involve provider credentials or external services; keep provider configuration outside package internals and validate through MCP/config tests.", + "owner": "SylphxAI/coderag", + "target": "next embedding-provider or MCP config change" + }, + { + "id": "package-release-forward-fix", + "description": "Published package versions and MCP behavior are forward-only for consumers; add release recovery/runbook proof before production lifecycle promotion.", + "owner": "SylphxAI/coderag", + "target": "before production lifecycle promotion" + } + ] + } } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 20e3cf1..6129f0e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,14 +23,38 @@ jobs: - name: Install dependencies run: bun install --frozen-lockfile + - name: Set up Rust + uses: dtolnay/rust-toolchain@stable + + - name: Build Rust retrieval core + run: cargo build --release + + - name: Rust core tests + run: cargo test + + - name: Golden retrieval evals + run: bun test test/golden-retrieval.test.ts + - name: Type check run: bun run type-check + - name: Build packages + run: bun run build + - name: Run tests run: bun run test - - name: Build packages - run: bun run build + - name: Public surface audit (README discovery) + run: bun run test:public-surface + + - name: Public benchmark proof + run: bun run benchmark:public-proof + + - name: Release gate + run: bun run benchmark:release-gate + + - name: Docs build + run: bun run docs:build - name: Upload coverage uses: codecov/codecov-action@v4 diff --git a/.gitignore b/.gitignore index 00971bb..48443e2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ node_modules/ +target/ dist/ *.log .DS_Store diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..cab3813 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,995 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.2.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5d6cac793997bd970000024b2934968efe83b382de4fdcf4fcb46b6ee4ad996" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "coderag-cli" +version = "0.1.0" +dependencies = [ + "coderag-core", + "serde", + "serde_json", +] + +[[package]] +name = "coderag-core" +version = "0.1.0" +dependencies = [ + "regex", + "serde", + "serde_json", + "sha2", + "walkdir", +] + +[[package]] +name = "coderag-mcp-server" +version = "0.1.0" +dependencies = [ + "anyhow", + "coderag-core", + "rmcp", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mio" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "regex" +version = "1.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rmcp" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4c9c94680f75470ee8083a0667988b5d7b5beb70b9f998a8e51de7c682ce60" +dependencies = [ + "async-trait", + "base64", + "chrono", + "futures", + "pastey", + "pin-project-lite", + "rmcp-macros", + "schemars", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "rmcp-macros" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90c23c8f26cae4da838fbc3eadfaecf2d549d97c04b558e7bd90526a9c28b42a" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "serde_json", + "syn", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schemars" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" +dependencies = [ + "chrono", + "dyn-clone", + "ref-cast", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d115b50f4aaeea07e79c1912f645c7513d81715d0420f8bc77a18c6260b307f" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_derive_internals" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..8b32fde --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,12 @@ +[workspace] +members = [ + "crates/coderag-core", + "crates/coderag-cli", + "crates/coderag-mcp-server", +] +resolver = "2" + +[workspace.package] +edition = "2021" +license = "MIT" +repository = "https://github.com/SylphxAI/coderag" \ No newline at end of file diff --git a/README.md b/README.md index 4df3be3..96ae611 100644 --- a/README.md +++ b/README.md @@ -2,474 +2,263 @@ # CodeRAG -**Lightning-fast hybrid code search for AI assistants** +### Your agent searched the codebase. **Did it find the right code?** + +Local-first hybrid code search for AI assistants. One MCP call indexes your repo +and returns **semantic chunks** — functions, classes, and methods — not noisy +grep dumps or slow cloud pipelines. [![npm version](https://img.shields.io/npm/v/@sylphx/coderag?style=flat-square&label=core)](https://www.npmjs.com/package/@sylphx/coderag) [![npm version](https://img.shields.io/npm/v/@sylphx/coderag-mcp?style=flat-square&label=mcp)](https://www.npmjs.com/package/@sylphx/coderag-mcp) [![CI](https://img.shields.io/github/actions/workflow/status/SylphxAI/coderag/ci.yml?style=flat-square)](https://github.com/SylphxAI/coderag/actions) [![License](https://img.shields.io/badge/License-MIT-blue?style=flat-square)](LICENSE) -**Zero dependencies** • **<50ms search** • **Hybrid TF-IDF + Vector** • **MCP ready** +**Local-first** · **MCP-first** · **Hybrid TF-IDF + Vector** · **~200 tests** · **Reproducible benchmark** -[Quick Start](#-quick-start) • [Features](#-features) • [MCP Setup](#-mcp-server-setup) • [Roadmap](docs/roadmap/sota-family-roadmap.md) • [API](#-api-reference) +[⭐ Star this repo](https://github.com/SylphxAI/coderag) if agents should find code with evidence, not guess from keyword hits. +· [Quick start](#quick-start) · [See it work](#see-it-work) · [Why not grep alone?](#why-not-grep-alone) +· [Roadmap](docs/roadmap/sota-family-roadmap.md) --- -## Why CodeRAG? +## The problem -Traditional code search tools are either **slow** (full-text grep), **inaccurate** (keyword matching), or **complex** (require external services). +Agents search codebases thousands of times per session. Most paths give you one +of two bad outcomes: -CodeRAG is different: +1. **grep/ripgrep** — fast, but literal. Misses `authenticateUser` when you ask + for "login flow". Returns whole files, not the function you need. +2. **Cloud RAG** — semantic, but needs Docker, vector DBs, embedding APIs, and + 10–30s cold starts before the first search. -``` -❌ Old way: Docker + ChromaDB + Ollama + 30 second startup -✅ CodeRAG: npx @sylphx/coderag-mcp (instant) -``` +The model still guesses which snippet matters. Wrong chunk → wrong patch → wasted +context. -| Feature | grep/ripgrep | Cloud RAG | CodeRAG | -|---------|-------------|-----------|---------| -| **Semantic understanding** | ❌ | ✅ | ✅ | -| **Zero external deps** | ✅ | ❌ | ✅ | -| **Offline support** | ✅ | ❌ | ✅ | -| **Startup time** | Instant | 10-30s | <1s | -| **Search latency** | ~100ms | ~500ms | <50ms | +**CodeRAG is built for the moment your agent needs the right code block, not a +directory of keyword hits.** ---- +## Why not grep alone? -## ✨ Features - -### Search -- 🔍 **Hybrid Search** - TF-IDF + optional vector embeddings -- 🧠 **StarCoder2 Tokenizer** - Code-aware tokenization (4.7MB, trained on code) -- 📊 **Smoothed IDF** - No term gets ignored, stable ranking -- ⚡ **<50ms Latency** - Instant results even on large codebases - -### Indexing -- 🚀 **1000-2000 files/sec** - Fast initial indexing -- 💾 **SQLite Persistence** - Instant startup (<100ms) with cached index -- ⚡ **Incremental Updates** - Smart diff detection, no full rebuilds -- 👁️ **File Watching** - Real-time index updates on file changes +| | grep/ripgrep | Cloud RAG | CodeRAG | +| --- | --- | --- | --- | +| **Semantic understanding** | ❌ Literal match | ✅ Embeddings | ✅ TF-IDF + optional vectors | +| **Zero external deps** | ✅ | ❌ Vector DB + embed API | ✅ Local by default | +| **Offline support** | ✅ | ❌ | ✅ | +| **Result shape** | Whole files / lines | Often whole files | AST chunks (functions, classes) | +| **Agent setup** | Shell tool | Docker + services | `npx @sylphx/coderag-mcp` | -### Integration -- 📦 **MCP Server** - Works with Claude Desktop, Cursor, VS Code, Windsurf -- 🧠 **Vector Search** - Optional OpenAI embeddings for semantic search -- 🌳 **AST Chunking** - Smart code splitting using [Synth](https://github.com/SylphxAI/synth) parsers (15+ languages) -- 💻 **Low Memory Mode** - SQL-based search for resource-constrained environments +Search latency and indexing throughput: reproduce with +[`bun run benchmark:public-proof`](#benchmark-proof) — do not trust hand-waved +ms claims. ---- +Full comparison: [how search works](docs/guide/how-search-works.md). -## 🚀 Quick Start +## See it work -### Option 1: MCP Server (Recommended for AI Assistants) +**Install once. Point at your repo.** ```bash -npx @sylphx/coderag-mcp --root=/path/to/project +claude mcp add coderag -- npx @sylphx/coderag-mcp --root=/absolute/path/to/project ``` -Or add to your MCP config: +Search with the `codebase_search` tool: ```json { - "mcpServers": { - "coderag": { - "command": "npx", - "args": ["-y", "@sylphx/coderag-mcp", "--root=/path/to/project"] - } - } + "query": "user authentication login", + "limit": 5, + "file_extensions": [".ts", ".tsx"], + "exclude_paths": ["node_modules", "dist"] } ``` -See [MCP Server Setup](#-mcp-server-setup) for Claude Desktop, Cursor, VS Code, etc. - -### Option 2: As a Library +Returns ranked chunks — not entire files: -```bash -npm install @sylphx/coderag -# or -bun add @sylphx/coderag -``` +```markdown +# Search: "user authentication login" (3 results) +## src/auth/login.ts:1-12 ```typescript -import { CodebaseIndexer, PersistentStorage } from '@sylphx/coderag' - -// Create indexer with persistent storage -const storage = new PersistentStorage({ codebaseRoot: './my-project' }) -const indexer = new CodebaseIndexer({ - codebaseRoot: './my-project', - storage, -}) - -// Index codebase (instant on subsequent runs) -await indexer.index({ watch: true }) - -// Search -const results = await indexer.search('authentication logic', { limit: 10 }) -console.log(results) -// [{ path: 'src/auth/login.ts', score: 0.87, matchedTerms: ['authentication', 'logic'], snippet: '...' }] -``` - ---- - -## 📦 Packages - -| Package | Description | Install | -|---------|-------------|---------| -| [@sylphx/coderag](./packages/core) | Core search library | `npm i @sylphx/coderag` | -| [@sylphx/coderag-mcp](./packages/mcp-server) | MCP server for AI assistants | `npx @sylphx/coderag-mcp` | - ---- - -## 🔌 MCP Server Setup - -### Claude Desktop - -Add to `claude_desktop_config.json`: - -**macOS**: `~/Library/Application Support/Claude/claude_desktop_config.json` -**Windows**: `%APPDATA%\Claude\claude_desktop_config.json` - -```json -{ - "mcpServers": { - "coderag": { - "command": "npx", - "args": ["-y", "@sylphx/coderag-mcp", "--root=/path/to/project"] - } - } +export async function authenticate(username: string, password: string) { + const user = await findUserByEmail(username) + return validatePassword(user, password) } ``` +``` -### Cursor - -Add to `~/.cursor/mcp.json` (macOS) or `%USERPROFILE%\.cursor\mcp.json` (Windows): +## Why agents use it -```json -{ - "mcpServers": { - "coderag": { - "command": "npx", - "args": ["-y", "@sylphx/coderag-mcp", "--root=/path/to/project"] - } - } -} -``` +| Need | What you get | +| --- | --- | +| Find implementation | AST chunks at semantic boundaries (functions, classes, methods) | +| Keyword + meaning | Hybrid TF-IDF with optional OpenAI embeddings | +| Fast iteration | Local index, incremental updates, file watching | +| Low setup | MCP server via `npx` — no Docker or ChromaDB required | +| Ship with proof | ~200 tests, reproducible public benchmark script | -### VS Code +## Quick Start -Add to VS Code settings (JSON) or `.vscode/mcp.json`: +### Claude Code (recommended) -```json -{ - "mcp": { - "servers": { - "coderag": { - "command": "npx", - "args": ["-y", "@sylphx/coderag-mcp", "--root=${workspaceFolder}"] - } - } - } -} +```bash +claude mcp add coderag -- npx @sylphx/coderag-mcp --root=/absolute/path/to/project ``` -### Windsurf +### Claude Desktop -Add to `~/.codeium/windsurf/mcp_config.json`: +Add to `claude_desktop_config.json`: ```json { "mcpServers": { "coderag": { "command": "npx", - "args": ["-y", "@sylphx/coderag-mcp", "--root=/path/to/project"] + "args": ["-y", "@sylphx/coderag-mcp", "--root=/absolute/path/to/project"] } } } ``` -### Claude Code +### Any MCP Client ```bash -claude mcp add coderag -- npx -y @sylphx/coderag-mcp --root=/path/to/project -``` - ---- - -## 🛠️ MCP Tool: `codebase_search` - -Search project source files with hybrid TF-IDF + vector ranking. - -### Parameters - -| Parameter | Type | Required | Default | Description | -|-----------|------|----------|---------|-------------| -| `query` | string | Yes | - | Search query | -| `limit` | number | No | 10 | Max results | -| `include_content` | boolean | No | true | Include code snippets | -| `file_extensions` | string[] | No | - | Filter by extension (e.g., `[".ts", ".tsx"]`) | -| `path_filter` | string | No | - | Filter by path pattern | -| `exclude_paths` | string[] | No | - | Exclude paths (e.g., `["node_modules", "dist"]`) | - -### Example - -```json -{ - "query": "user authentication login", - "limit": 5, - "file_extensions": [".ts", ".tsx"], - "exclude_paths": ["node_modules", "dist", "test"] -} -``` - -### Response Format - -LLM-optimized output (minimal tokens, maximum content): - -```markdown -# Search: "user authentication login" (3 results) - -## src/auth/login.ts:15-28 -```typescript -15: export async function authenticate(credentials) { -16: const user = await findUser(credentials.email) -17: return validatePassword(user, credentials.password) -18: } +npx @sylphx/coderag-mcp --root=/absolute/path/to/project ``` -## src/middleware/auth.ts:42-55 [md→typescript] -```typescript -42: // Embedded code from markdown docs -43: const authMiddleware = (req, res, next) => { -``` - -## src/utils/large.ts:1-200 [truncated] -```typescript -1: // First 70% shown... +Need Cursor, VS Code, Windsurf, or library usage? See the +[installation guide](docs/guide/installation.md) and [MCP docs](docs/mcp/overview.md). -... [800 chars truncated] ... +### As a Library -195: // Last 20% shown -``` +```bash +bun add @sylphx/coderag ``` ---- - -## 📚 API Reference - -### `CodebaseIndexer` - -Main class for indexing and searching. - ```typescript import { CodebaseIndexer, PersistentStorage } from '@sylphx/coderag' -const storage = new PersistentStorage({ codebaseRoot: './project' }) -const indexer = new CodebaseIndexer({ - codebaseRoot: './project', - storage, - maxFileSize: 1024 * 1024, // 1MB default -}) +const storage = new PersistentStorage({ codebaseRoot: './my-project' }) +const indexer = new CodebaseIndexer({ codebaseRoot: './my-project', storage }) -// Index with file watching await indexer.index({ watch: true }) - -// Search with options -const results = await indexer.search('query', { - limit: 10, - includeContent: true, - fileExtensions: ['.ts', '.js'], - excludePaths: ['node_modules'], -}) - -// Stop watching -await indexer.stopWatch() +const results = await indexer.search('authentication logic', { limit: 10 }) ``` -### `PersistentStorage` +--- -SQLite-backed storage for instant startup. +## MCP Tool: `codebase_search` -```typescript -import { PersistentStorage } from '@sylphx/coderag' +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `query` | string | — | Search query (required) | +| `limit` | number | 10 | Max results | +| `include_content` | boolean | true | Include code snippets | +| `file_extensions` | string[] | — | Filter by extension | +| `path_filter` | string | — | Filter by path pattern | +| `exclude_paths` | string[] | — | Exclude paths | -const storage = new PersistentStorage({ - codebaseRoot: './project', // Creates .coderag/ folder - dbPath: './custom.db', // Optional custom path -}) -``` +Full tool reference: [docs/mcp/tools.md](docs/mcp/tools.md). -### Low-Level TF-IDF Functions - -```typescript -import { buildSearchIndex, searchDocuments, initializeTokenizer } from '@sylphx/coderag' +--- -// Initialize StarCoder2 tokenizer (4.7MB, one-time download) -await initializeTokenizer() +## Benchmark Proof -// Build index -const documents = [ - { uri: 'file://auth.ts', content: 'export function authenticate...' }, - { uri: 'file://user.ts', content: 'export class User...' }, -] -const index = await buildSearchIndex(documents) +Performance claims in this README are backed only by the checked-in public +benchmark script — not hand-waved marketing numbers. -// Search -const results = await searchDocuments('authenticate user', index, { limit: 5 }) +```bash +bun run benchmark:public-proof ``` -### Vector Search (Optional) +The script indexes `fixtures/benchmark-corpus/` (30 TypeScript files), runs +hybrid TF-IDF search 20 times (3 warmup), and prints indexing throughput plus +search p50/min/max latency. -For semantic search with embeddings: - -```typescript -import { hybridSearch, createEmbeddingProvider } from '@sylphx/coderag' - -// Requires OPENAI_API_KEY environment variable -const results = await hybridSearch('authentication flow', indexer, { - vectorWeight: 0.7, // 70% vector, 30% TF-IDF - limit: 10, -}) -``` +See [benchmark proof](docs/benchmark.md) for methodology and latest reproduced +results. --- -## ⚙️ Configuration - -### MCP Server Options +## Packages -| Option | Default | Description | -|--------|---------|-------------| -| `--root=` | Current directory | Codebase root path | -| `--max-size=` | 1048576 (1MB) | Max file size to index | -| `--no-auto-index` | false | Disable auto-indexing on startup | - -### Environment Variables - -| Variable | Description | -|----------|-------------| -| `OPENAI_API_KEY` | Enable vector search with OpenAI embeddings | -| `OPENAI_BASE_URL` | Custom OpenAI-compatible endpoint | -| `EMBEDDING_MODEL` | Embedding model (default: `text-embedding-3-small`) | -| `EMBEDDING_DIMENSIONS` | Custom embedding dimensions | +| Package | Description | Install | +| --- | --- | --- | +| [@sylphx/coderag](packages/core) | Core search library | `npm i @sylphx/coderag` | +| [@sylphx/coderag-mcp](packages/mcp-server) | MCP server for AI assistants | `npx @sylphx/coderag-mcp` | --- -## 📊 Performance - -| Metric | Value | -|--------|-------| -| **Initial indexing** | ~1000-2000 files/sec | -| **Startup with cache** | <100ms | -| **Search latency** | <50ms | -| **Memory per 1000 files** | ~1-2 MB | -| **Tokenizer size** | 4.7MB (StarCoder2) | +## Documentation -### Benchmarks - -Tested on MacBook Pro M1, 16GB RAM: - -| Codebase | Files | Index Time | Search Time | -|----------|-------|------------|-------------| -| Small (100 files) | 100 | 0.5s | <10ms | -| Medium (1000 files) | 1,000 | 2s | <30ms | -| Large (10000 files) | 10,000 | 15s | <50ms | +| Topic | Link | +| --- | --- | +| Docs site | [coderag.sylphx.com](https://coderag.sylphx.com) | +| Getting started | [docs/guide/getting-started.md](docs/guide/getting-started.md) | +| MCP server | [docs/mcp/overview.md](docs/mcp/overview.md) | +| How search works | [docs/guide/how-search-works.md](docs/guide/how-search-works.md) | +| Benchmark proof | [docs/benchmark.md](docs/benchmark.md) | +| Stop code-search guessing | [docs/articles/stop-code-search-guessing.md](docs/articles/stop-code-search-guessing.md) | +| API reference | [docs/api/overview.md](docs/api/overview.md) | --- -## 🏗️ Architecture - -``` -coderag/ -├── packages/ -│ ├── core/ # @sylphx/coderag -│ │ ├── src/ -│ │ │ ├── indexer.ts # Main indexer with file watching -│ │ │ ├── tfidf.ts # TF-IDF with StarCoder2 tokenizer -│ │ │ ├── code-tokenizer.ts # StarCoder2 tokenization -│ │ │ ├── hybrid-search.ts # Vector + TF-IDF fusion -│ │ │ ├── incremental-tfidf.ts # Smart incremental updates -│ │ │ ├── storage-persistent.ts # SQLite storage -│ │ │ ├── vector-storage.ts # LanceDB vector storage -│ │ │ ├── embeddings.ts # OpenAI embeddings -│ │ │ ├── ast-chunking.ts # Synth AST chunking -│ │ │ └── language-config.ts # Language registry (15+ languages) -│ │ └── package.json -│ │ -│ └── mcp-server/ # @sylphx/coderag-mcp -│ ├── src/ -│ │ └── index.ts # MCP server -│ └── package.json -``` - -### How It Works - -1. **Indexing**: Scans codebase, tokenizes with StarCoder2, builds TF-IDF index -2. **AST Chunking**: Splits code at semantic boundaries (functions, classes, etc.) -3. **Storage**: Persists to SQLite (`.coderag/` folder) for instant startup -4. **Watching**: Detects file changes, performs incremental updates -5. **Search**: Hybrid TF-IDF + optional vector search with score fusion - -### Supported Languages - -AST-based chunking with semantic boundary detection: +## Security model -| Category | Languages | -|----------|-----------| -| **JavaScript** | JavaScript, TypeScript, JSX, TSX | -| **Systems** | Python, Go, Java, C | -| **Markup** | Markdown, HTML, XML | -| **Data/Config** | JSON, YAML, TOML, INI | -| **Other** | Protobuf | +- **Root confinement** — `--root` pins indexing and search to one repository tree. +- **Exclude paths** — `exclude_paths` and default ignores skip `node_modules`, build output, and VCS metadata. +- **Local-first** — TF-IDF indexing runs on your machine; embeddings are optional and caller-configured. +- **Evidence fields** — results include file path, line range, symbol, score route, and index freshness for verification. -**Embedded Code Support**: Automatically parses code blocks in Markdown and `