From d127527feb284adaf68ccf2b6c86fdaa42d11e2c Mon Sep 17 00:00:00 2001 From: Kenny Rch Date: Thu, 9 Jul 2026 19:26:49 +0300 Subject: [PATCH] feat: Implement CLI functionality for user query processing --- .gitignore | 1 + src/docs_buddy/_vendor/__init__.py | 0 src/docs_buddy/_vendor/dotconfig.py | 75 +++++++++ src/docs_buddy/adapters/__init__.py | 54 ++++-- src/docs_buddy/adapters/agent.py | 47 ++++-- src/docs_buddy/adapters/whoosh_index.py | 17 +- src/docs_buddy/common.py | 9 + src/docs_buddy/entrypoints/bootstrap.py | 3 +- src/docs_buddy/entrypoints/cli/README.md | 15 +- src/docs_buddy/entrypoints/cli/__main__.py | 158 +++++++++++++----- .../cli/data/docs_buddy_template.yaml | 51 ++++++ .../entrypoints/cli/requirements.txt | 2 + src/docs_buddy/services/commands.py | 1 + src/docs_buddy/services/events.py | 1 + src/docs_buddy/services/handlers.py | 4 +- src/docs_buddy/services/use_cases.py | 10 +- tests/integration/test_adapters.py | 11 +- tests/unit/test_common.py | 26 +++ tests/unit/test_services.py | 24 ++- tests/unit/test_tools.py | 12 +- todo.md | 34 ++-- 21 files changed, 445 insertions(+), 110 deletions(-) create mode 100644 src/docs_buddy/_vendor/__init__.py create mode 100644 src/docs_buddy/_vendor/dotconfig.py create mode 100644 src/docs_buddy/entrypoints/cli/data/docs_buddy_template.yaml create mode 100644 src/docs_buddy/entrypoints/cli/requirements.txt create mode 100644 tests/unit/test_common.py diff --git a/.gitignore b/.gitignore index 8a9f107..ae78a98 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ __pycache__ .whoosh bin/ +.docs_buddy.yaml diff --git a/src/docs_buddy/_vendor/__init__.py b/src/docs_buddy/_vendor/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/docs_buddy/_vendor/dotconfig.py b/src/docs_buddy/_vendor/dotconfig.py new file mode 100644 index 0000000..b634caf --- /dev/null +++ b/src/docs_buddy/_vendor/dotconfig.py @@ -0,0 +1,75 @@ +"""dotconfig - Create config objects from JSON/YAML""" + +import json +from pathlib import Path + +import yaml + + +class Config: + """Wrapper for nested config with dot-notation access. + + >>> config = Config({"database": {"host": "localhost", "port": 5432}}) + >>> config.database.host + 'localhost' + >>> config.database.port + 5432 + + Supports loading JSON/YAML + """ + + def __new__(cls, config): + if isinstance(config, list): + return [cls(c) if isinstance(c, (list, dict)) else c for c in config] + return super().__new__(cls) + + def __init__(self, config: dict): + self._config = config + + @classmethod + def from_json(cls, file): + """Load from JSON file""" + data = cls._get_data(file, json.load) + return cls(data) + + @classmethod + def from_yaml(cls, file): + """Load from YAML file""" + data = cls._get_data(file, yaml.safe_load) + return cls(data) + + @staticmethod + def _get_data(source, loader): + """Load data from file object or path""" + try: + filepath = Path(source) + except TypeError: + # treat as filelike + data = loader(source) + else: + with open(filepath) as f: + data = loader(f) + return data + + def __getattr__(self, name): + try: + value = self._config[name] + except KeyError as e: + raise AttributeError(f"No config '{name}' found.") from e + + if isinstance(value, (dict, list)): + return Config(value) + return value + + def __getitem__(self, name): + value = self._config[name] + if isinstance(value, (list, dict)): + return Config(value) + return value + + def __str__(self): + return json.dumps(self._config) + + def __repr__(self): + cls_name = type(self).__name__ + return f"{cls_name}({self._config!r})" diff --git a/src/docs_buddy/adapters/__init__.py b/src/docs_buddy/adapters/__init__.py index d5a66f0..6a09683 100644 --- a/src/docs_buddy/adapters/__init__.py +++ b/src/docs_buddy/adapters/__init__.py @@ -25,6 +25,10 @@ class MemoryBusError(DocsBuddyError): pass +class FileSystemRepoStorageError(DocsBuddyError): + pass + + class InMemoryMessageBus: """In memory implementation of the message bus protocol""" @@ -75,10 +79,11 @@ def is_already_cloned(self) -> bool: def can_clone(self) -> bool: return self.fake_can_clone - def clone_repo(self, url: str) -> None: - self.actions.append(("CLONE", url, self._target)) + def clone_repo(self, url: str, branch: str) -> None: + self.actions.append(("CLONE", url, self._target, branch)) - def pull_repo(self): + def pull_repo(self, branch: str) -> None: + self.actions.append(("CHECKOUT", branch)) self.actions.append(("PULL",)) @@ -95,11 +100,19 @@ def __repr__(self): def is_already_cloned(self) -> bool: return self._is_git_repository(self._target) - def pull_repo(self): - self._update_repository(self._target) + def pull_repo(self, branch: str) -> None: + try: + self._update_repository(self._target, branch) + except subprocess.CalledProcessError as exc: + msg = f"Unable to pull branch '{branch}'. Error: '{exc.stderr}'" + raise FileSystemRepoStorageError(msg) from exc - def clone_repo(self, url: str) -> None: - self._clone_repository(url, self._target) + def clone_repo(self, url: str, branch: str) -> None: + try: + self._clone_repository(url, self._target, branch) + except subprocess.CalledProcessError as exc: + msg = f"Unable to clone repository '{url}'. Error: '{exc.stderr}'" + raise FileSystemRepoStorageError(msg) from exc def can_clone(self): """Indicates whether we can clone a new repo in the target @@ -123,18 +136,37 @@ def _is_git_repository(directory: str) -> bool: return git_dir.exists() and git_dir.is_dir() @staticmethod - def _clone_repository(url: str, target_dir: str) -> None: + def _clone_repository(url: str, target_dir: str, branch: str) -> None: """Clone a git repository with depth 1.""" subprocess.run( - ["git", "clone", "--depth", "1", url, target_dir], + ["git", "clone", "--depth", "1", "--branch", branch, url, target_dir], check=True, capture_output=True, ) @staticmethod - def _update_repository(directory: str) -> None: + def _update_repository(directory: str, branch: str) -> None: """Pull latest changes from a git repository.""" - subprocess.run(["git", "pull"], cwd=directory, check=True, capture_output=True) + subprocess.run( + [ + "git", + "fetch", + "--depth", + "1", + "origin", + f"{branch}:refs/remotes/origin/{branch}", + ], + cwd=directory, + check=True, + capture_output=True, + ) + + subprocess.run( + ["git", "checkout", "-B", branch, f"origin/{branch}"], + cwd=directory, + capture_output=True, + check=True, + ) class FakeIntermediateStorage: diff --git a/src/docs_buddy/adapters/agent.py b/src/docs_buddy/adapters/agent.py index ab29a6a..b397b6b 100644 --- a/src/docs_buddy/adapters/agent.py +++ b/src/docs_buddy/adapters/agent.py @@ -80,8 +80,17 @@ async def on_tool_end(self, context, agent, tool, result): ) -def make_search_tool(index: services.DocumentIndex) -> Callable: - """Creates a search tool over the index""" +def make_search_tool( + index: services.DocumentIndex, tool_id: str, content_information: str +) -> Callable: + """Creates a search tool over the index + + @arg index - the index to search over + @arg tool_id - the unique identifier used to create the tool name + @arg content_information - a description of content the tool searches + + @returns - a search tool over the index + """ @log_input(log, logging.INFO) def search_document_index(phrase: str, max_results: int = 5) -> list[str]: @@ -95,6 +104,10 @@ def search_document_index(phrase: str, max_results: int = 5) -> list[str]: list[str]: A list of JSON formatted strings representing the the results. Each result has associated content, path and metadata fields + + What is indexed: + + {content_information} """ if not max_results > 0: @@ -106,6 +119,12 @@ def search_document_index(phrase: str, max_results: int = 5) -> list[str]: results = index.search(query, max_results) return [str(r) for r in results] + search_document_index.__doc__ = search_document_index.__doc__.format( + content_information=content_information + ) + + search_document_index.__name__ = search_document_index.__name__ + f"_{tool_id}" + return search_document_index @@ -152,28 +171,26 @@ def generate_final_response( return final_response -def make_openai_research_agent(prompt: str) -> services.Agent: +def make_openai_research_agent( + prompt: str, model_name: str, base_url: str +) -> services.Agent: """Creates an openai agent""" - BASE_URL = os.getenv("OPENAI_BASE_URL") or "" - API_KEY = os.getenv("OPENAI_API_KEY") or "" - MODEL_NAME = os.getenv("DOCS_BUDDY_MODEL_NAME") or "" + api_key = os.getenv("OPENAI_API_KEY") - if not BASE_URL or not API_KEY or not MODEL_NAME: - raise AgentError( - "Please set OPENAI_BASE_URL, OPENAI_API_KEY, DOCS_BUDDY_MODEL_NAME" - ) + if not api_key: + raise AgentError("Please set the OPENAI_API_KEY environment variable") - client = openai.AsyncOpenAI(base_url=BASE_URL, api_key=API_KEY) + client = openai.AsyncOpenAI(base_url=base_url, api_key=api_key) agents.set_tracing_disabled(disabled=True) class CustomModelProvider(agents.ModelProvider): - def get_model(self, model_name: str | None) -> agents.Model: + def get_model(self, name: str | None) -> agents.Model: return agents.OpenAIChatCompletionsModel( - model=model_name or MODEL_NAME, openai_client=client + model=name or model_name, openai_client=client ) - CUSTOM_MODEL_PROVIDER = CustomModelProvider() + custom_model_provider = CustomModelProvider() def openai_agent( query: domain.Query, tools: list[Callable] @@ -205,7 +222,7 @@ def openai_agent( agent, str(query), LoggingHooks(), - agents.RunConfig(model_provider=CUSTOM_MODEL_PROVIDER), + agents.RunConfig(model_provider=custom_model_provider), ) ) diff --git a/src/docs_buddy/adapters/whoosh_index.py b/src/docs_buddy/adapters/whoosh_index.py index 571b01b..daa8b4e 100644 --- a/src/docs_buddy/adapters/whoosh_index.py +++ b/src/docs_buddy/adapters/whoosh_index.py @@ -27,11 +27,14 @@ class WhooshDocumentIndex: ) _SEARCH_FIELDS = ["content", "metadata", "path_keywords"] - def __init__(self, index_location: PathLike | None = None): + def __init__( + self, index_location: PathLike | None = None, file_prefix: str | None = None + ): """ Initialize a Whoosh document index. """ + self._file_prefix = file_prefix self._index = None if index_location: self._index = index.open_dir(index_location) @@ -83,6 +86,15 @@ def search(self, query: domain.Query, max_results: int) -> list[domain.QueryResu parsed_query = self._query_parser.parse(str(query)) + prefix = "" + + if self._file_prefix: + prefix = ( + self._file_prefix + "/" + if not self._file_prefix.endswith("/") + else self._file_prefix + ) + with self._index.searcher() as searcher: # todo: consider interaction between indexing and searching # is locking required for coordination? @@ -90,9 +102,10 @@ def search(self, query: domain.Query, max_results: int) -> list[domain.QueryResu results = [ domain.QueryResult( content=r["content"], - path=r["path"], + path=prefix + r["path"], metadata=json.loads(r["metadata"]), ) for r in results ] + return results diff --git a/src/docs_buddy/common.py b/src/docs_buddy/common.py index e865853..5a2daae 100644 --- a/src/docs_buddy/common.py +++ b/src/docs_buddy/common.py @@ -8,6 +8,7 @@ from typing import TypeAlias import logging import functools +import re PathLike: TypeAlias = str | os.PathLike @@ -45,3 +46,11 @@ def wrapper(*args, **kwargs): return wrapper return decorator + + +def sanitize_to_python_id(identifier: str) -> str: + """Converts string to valid Python identifier""" + sanitized = re.sub(r"[^a-zA-Z0-9_]", "_", identifier) + if sanitized[0].isdigit(): + sanitized = "name_" + sanitized + return sanitized diff --git a/src/docs_buddy/entrypoints/bootstrap.py b/src/docs_buddy/entrypoints/bootstrap.py index d634dd9..4582748 100644 --- a/src/docs_buddy/entrypoints/bootstrap.py +++ b/src/docs_buddy/entrypoints/bootstrap.py @@ -17,6 +17,7 @@ def get_message_bus( repository_storage_path: PathLike, chunks_storage_path: PathLike, lexical_index_storage_path: PathLike, + doc_extensions: tuple[str, ...], ) -> handlers.MessageBus: """Create and configure the application message bus with all dependencies. @@ -31,7 +32,7 @@ def get_message_bus( message_bus = adapters.InMemoryMessageBus() repo_storage = adapters.FileSystemRepoStorage(repository_storage_path) docs_storage = adapters.FileSystemDocsStorage( - repository_storage_path, chunks_storage_path + repository_storage_path, chunks_storage_path, doc_extensions ) chunks_pipeline = adapters.FileSystemDocumentChunksPipeline( chunks_storage_path, lexical_index_storage_path diff --git a/src/docs_buddy/entrypoints/cli/README.md b/src/docs_buddy/entrypoints/cli/README.md index 7af6621..16c88b6 100644 --- a/src/docs_buddy/entrypoints/cli/README.md +++ b/src/docs_buddy/entrypoints/cli/README.md @@ -6,15 +6,20 @@ from the terminal. ## Usage ```bash -usage: python -m docs_buddy.entrypoints.cli [-h] [--repo-id REPO_IDS] [--update-sources | query] +usage: __main__.py [-h] [--log-level {DEBUG,INFO,WARNING,ERROR,CRITICAL}] [--update-sources] [--init] + [--current-config] + [query] Docs Buddy CLI positional arguments: - query Query string to search the documentation + query Query string to search the documentation options: - -h, --help show this help message and exit - --repo-id REPO_IDS Repository ID(s). Can be used multiple times, each time for a different ID - --update-sources Update document sources from repository and recreate index + -h, --help show this help message and exit + --log-level {DEBUG,INFO,WARNING,ERROR,CRITICAL} + The log level to use (default WARNING) + --update-sources Update document sources from repository and recreate index + --init Create a .docs_buddy.yaml config file from template in the current directory + --current-config Show location of the current config file in use ``` diff --git a/src/docs_buddy/entrypoints/cli/__main__.py b/src/docs_buddy/entrypoints/cli/__main__.py index c8424d7..c67bf79 100644 --- a/src/docs_buddy/entrypoints/cli/__main__.py +++ b/src/docs_buddy/entrypoints/cli/__main__.py @@ -7,12 +7,16 @@ import sys import textwrap -from docs_buddy import adapters, services, domain +from docs_buddy import adapters, services, domain, common from docs_buddy.services import commands from docs_buddy.entrypoints import bootstrap +from docs_buddy._vendor.dotconfig import Config log = logging.getLogger(__name__) +CONFIG_FILE_NAME = ".docs_buddy.yaml" +DEFAULT_FILE_EXTENSIONS = ("mdx", "md") + def _configure_logging(log_level: int) -> None: logging.basicConfig( @@ -32,19 +36,53 @@ def _get_data_dir() -> Path: return base / "docs-buddy" +def _init_config() -> None: + """Create a .docs_buddy.yaml config file from the template in the current directory.""" + template = Path(__file__).parent / "data" / "docs_buddy_template.yaml" + target = Path.cwd() / CONFIG_FILE_NAME + if target.exists(): + print(f"Config file already exists: {target}", file=sys.stderr) + sys.exit(1) + target.write_text(template.read_text()) + print(f"Created config file: {target}. Update the git source(s)") + + +def _find_config() -> Path | None: + """Walk up from cwd to home, return first .docs_buddy.yaml found.""" + current = Path.cwd().resolve() + home = Path.home().resolve() + while current >= home: + candidate = current / CONFIG_FILE_NAME + if candidate.exists(): + return candidate + if current == home: + break + current = current.parent + return None + + +def _load_config() -> Config: + """Load the config file found by _find_config.""" + config_path = _find_config() + if config_path is None: + print( + f"No {CONFIG_FILE_NAME} found. Run --init first.", + file=sys.stderr, + ) + sys.exit(1) + return Config.from_yaml(config_path) + + +def _derive_repo_id(repo_url: str) -> str: + """Strip protocol from URL to obtain a filesystem-friendly repo ID.""" + return repo_url.split("://", 1)[-1] + + def main() -> None: """CLI main entrypoint""" parser = argparse.ArgumentParser(description="Docs Buddy CLI") - parser.add_argument( - "--repo-id", - action="append", - dest="repo_ids", - default=[], - help="Repository ID(s). Can be used multiple times, each time for a different ID", - ) - parser.add_argument( "--log-level", default="WARNING", @@ -58,6 +96,16 @@ def main() -> None: action="store_true", help="Update document sources from repository and recreate index", ) + group.add_argument( + "--init", + action="store_true", + help="Create a .docs_buddy.yaml config file from template in the current directory", + ) + group.add_argument( + "--current-config", + action="store_true", + help="Show location of the current config file in use", + ) group.add_argument( "query", nargs="?", @@ -70,53 +118,85 @@ def main() -> None: log_level = getattr(logging, args.log_level) _configure_logging(log_level) - # Require at least one repository ID - if not args.repo_ids: - parser.error("at least one --repo-id is required") - data_dir = _get_data_dir() - if args.update_sources: - for repository_id in args.repo_ids: - website_url = f"https://github.com/{repository_id}.git" + if args.init: + _init_config() + elif args.current_config: + config_path = _find_config() + if config_path: + print(config_path) + else: + print( + f"No {CONFIG_FILE_NAME} found from current directory up to home.", + file=sys.stderr, + ) + sys.exit(1) + elif args.update_sources: + config = _load_config() + for entry in config.repositories: # type: ignore[attr-defined] + repo_url = entry.repo_url + branch = entry.branch + repo_id = _derive_repo_id(repo_url) + + try: + doc_extensions = entry.file_extensions + except AttributeError: + doc_extensions = DEFAULT_FILE_EXTENSIONS message_bus = bootstrap.get_message_bus( - repository_storage_path=data_dir / "repos" / repository_id, - chunks_storage_path=data_dir / "chunks" / repository_id, - lexical_index_storage_path=data_dir / "whoosh" / repository_id, + repository_storage_path=data_dir / "repos" / repo_id, + doc_extensions=tuple(doc_extensions), + chunks_storage_path=data_dir / "chunks" / repo_id, + lexical_index_storage_path=data_dir / "whoosh" / repo_id, ) - sync_repo_command = commands.SyncRepo(website_url) - message_bus.send(sync_repo_command) + sync_repo_command = commands.SyncRepo(repo_url, branch) + + try: + message_bus.send(sync_repo_command) + except common.DocsBuddyError as exc: + msg = f"Error encountered updating sources: {exc}" + log.exception(msg) + sys.exit(1) elif args.query: - # todo: Use the first repo ID for search (multi-repo search not yet implemented) - repo_id = args.repo_ids[0] + config = _load_config() + tools = [] + for repo_config in config.repositories: # type: ignore[attr-defined] + repo_url = repo_config.repo_url + repo_id = _derive_repo_id(repo_url) + + index_path = data_dir / "whoosh" / repo_id + if not index_path.exists(): + log.warning("No index found for %s. Skipping.", repo_id) + continue + + file_prefix = repo_config.formatted_file_prefix + document_index = adapters.WhooshDocumentIndex(index_path, file_prefix) + + content_description = repo_config.repo_content_description + tool_id = common.sanitize_to_python_id(repo_id) + tools.append( + adapters.make_search_tool(document_index, tool_id, content_description) + ) - index_path = data_dir / "whoosh" / repo_id - if not index_path.exists(): - log.error("No index found for %s. Run --update-sources first", repo_id) + if not tools: + log.error( + "No valid documentation indexes found. Please sync repositories first." + ) sys.exit(1) - document_index = adapters.WhooshDocumentIndex(index_location=index_path) - try: query = domain.Query(args.query) except domain.InvalidQueryError as exc: log.error("Invalid query detected: %s", exc) sys.exit(1) - base_url = f"https://github.com/{repo_id}/blob/main/" - - tools = [adapters.make_search_tool(document_index)] - - system_prompt = textwrap.dedent("""\ - You are a documentation assistant equipped to answer user queries - by searching the docs. - """) - try: - research_user_query = adapters.make_openai_research_agent(system_prompt) + research_user_query = adapters.make_openai_research_agent( + config.system_prompt, config.model_name, config.openai_base_url + ) except adapters.AgentError: log.exception("Something went wrong while configuring the agent") sys.exit(1) @@ -127,7 +207,7 @@ def main() -> None: print(response.answer) print("\nReferences:\n") for path in response.citations: - print(f"{base_url}{path}") + print(path + "\n") else: parser.print_help() diff --git a/src/docs_buddy/entrypoints/cli/data/docs_buddy_template.yaml b/src/docs_buddy/entrypoints/cli/data/docs_buddy_template.yaml new file mode 100644 index 0000000..dbd49cb --- /dev/null +++ b/src/docs_buddy/entrypoints/cli/data/docs_buddy_template.yaml @@ -0,0 +1,51 @@ +--- +model_name: "deepseek/deepseek-v4-flash" # model to use from selected provider +openai_base_url: "https://openrouter.ai/api/v1" # supports any provider with an openai compatible API + +# tweak system prompt to preference +system_prompt: >- + You are a documentation assistant equipped to answer user queries + by searching the documentation sources provided. + +repositories: + # One or more git repositories to clone + + # Github example: + # + # repo_url - the https URL to clone + # branch - the branch to clone master, main, trunk etc.. + # formatted_file_prefix - the prefix to each file in the repo loading its formatted version + # repo_content_description - Description of the repo's content + # file_extensions - list of extensions of files to select for search (Default if not specified: md, mdx) + + # Replace bracketed text with the correct values + - repo_url: "https://github.com/{owner}/{repo}.git" + branch: "main" + formatted_file_prefix: "https://github.com/{owner}/{repo}/blob/main" + repo_content_description: "API documentation for foo project" + file_extensions: + - md + - mdx + + # Gitlab example + + #- repo_url: "https://gitlab.com/{owner}/{repo}.git" + # branch: "master" + # formatted_file_prefix: "https://gitlab.com/{owner}/{repo}/-/blob/{branch}" + # repo_content_description: "API documentation for foo project" + + # Gitea example + + #- repo_url: "https://gitea.com/{owner}/{repo}.git" + # branch: "trunk" + # formatted_file_prefix: "https://gitea.com/{owner}/{repo}/src/branch/{branch}" + # repo_content_description: "API documentation for foo project" + + # Codeberg / Self-hosted Forgejo example + + #- repo_url: "https://codeberg.org/{owner}/{repo}.git" + # branch: "main" + # formatted_file_prefix: "https://codeberg.org/{owner}/{repo}/src/branch/{branch}" + # repo_content_description: "API documentation for foo project" + + # Follow a similar pattern for any other http git hosting variants e.g. sourceforge, self-hosted diff --git a/src/docs_buddy/entrypoints/cli/requirements.txt b/src/docs_buddy/entrypoints/cli/requirements.txt new file mode 100644 index 0000000..f4f10e1 --- /dev/null +++ b/src/docs_buddy/entrypoints/cli/requirements.txt @@ -0,0 +1,2 @@ +PyYAML==6.0.3 +types-PyYAML==6.0.12.20260518 diff --git a/src/docs_buddy/services/commands.py b/src/docs_buddy/services/commands.py index aa7954f..0f9c308 100644 --- a/src/docs_buddy/services/commands.py +++ b/src/docs_buddy/services/commands.py @@ -12,6 +12,7 @@ class SyncRepo(Command): """Trigger syncing of repository indicated by url""" url: str + branch: str @dataclass(frozen=True) diff --git a/src/docs_buddy/services/events.py b/src/docs_buddy/services/events.py index 2172caa..ec02b01 100644 --- a/src/docs_buddy/services/events.py +++ b/src/docs_buddy/services/events.py @@ -12,6 +12,7 @@ class RepositorySynced(Event): """Indicate that repository has been synced""" url: str + branch: str @dataclass(frozen=True) diff --git a/src/docs_buddy/services/handlers.py b/src/docs_buddy/services/handlers.py index a46b13f..e824e92 100644 --- a/src/docs_buddy/services/handlers.py +++ b/src/docs_buddy/services/handlers.py @@ -19,9 +19,9 @@ def sync_repository( ) -> None: """Trigger repository sync""" - use_cases.sync_repository(command.url, repo_storage) + use_cases.sync_repository(command.url, command.branch, repo_storage) - message_bus.publish(events.RepositorySynced(url=command.url)) + message_bus.publish(events.RepositorySynced(url=command.url, branch=command.branch)) message_bus.send(commands.UpdateDocumentArtifacts()) diff --git a/src/docs_buddy/services/use_cases.py b/src/docs_buddy/services/use_cases.py index afd1acd..bcec9a5 100644 --- a/src/docs_buddy/services/use_cases.py +++ b/src/docs_buddy/services/use_cases.py @@ -31,9 +31,9 @@ def is_already_cloned(self) -> bool: ... def can_clone(self) -> bool: ... - def pull_repo(self) -> None: ... + def pull_repo(self, branch: str) -> None: ... - def clone_repo(self, url: str) -> None: ... + def clone_repo(self, url: str, branch: str) -> None: ... class SupportsIntermediateStorage(Protocol): @@ -74,13 +74,13 @@ class DocumentChunksPipeline(SupportsIntermediateStorage, Protocol): def get_document_chunks(self) -> Iterator[domain.DocumentChunk]: ... -def sync_repository(url: str, storage: RepoStorage) -> None: +def sync_repository(url: str, branch: str, storage: RepoStorage) -> None: """Synchronizes a git repository to local storage""" if storage.is_already_cloned(): - storage.pull_repo() + storage.pull_repo(branch) elif storage.can_clone(): - storage.clone_repo(url) + storage.clone_repo(url, branch) else: raise RepositorySyncError("Unable to sync repository") diff --git a/tests/integration/test_adapters.py b/tests/integration/test_adapters.py index 7bf5009..af97e94 100644 --- a/tests/integration/test_adapters.py +++ b/tests/integration/test_adapters.py @@ -137,11 +137,16 @@ def test_can_search_whoosh_index() -> None: indexer.fit(iter(chunks), temp_dir) - document_index = adapters.WhooshDocumentIndex(temp_dir) + file_prefix = "https://github.com/akash-network/website/blob/main" + + document_index = adapters.WhooshDocumentIndex(temp_dir, file_prefix) query = domain.Query("providers") results = document_index.search(query, max_results=10) assert len(results) > 1 + # prefix is prepended to result paths + assert all([r.path.startswith(file_prefix) for r in results]) + # it should be possible to specify max length of results assert len(document_index.search(query, max_results=1)) == 1 @@ -161,7 +166,9 @@ def test_can_create_whoosh_search_tool() -> None: document_index = adapters.WhooshDocumentIndex(temp_dir) - search = adapters.make_search_tool(document_index) + search = adapters.make_search_tool( + document_index, "foo_docs", "Foo documentation" + ) search_phrase = "providers" results = search(search_phrase) diff --git a/tests/unit/test_common.py b/tests/unit/test_common.py new file mode 100644 index 0000000..c2a2325 --- /dev/null +++ b/tests/unit/test_common.py @@ -0,0 +1,26 @@ +from docs_buddy.common import sanitize_to_python_id + + +def test_normal_url_like(): + result = sanitize_to_python_id("github.com/programmer-ke/docs-buddy") + assert result == "github_com_programmer_ke_docs_buddy" + + +def test_already_clean(): + result = sanitize_to_python_id("my_repo_123") + assert result == "my_repo_123" + + +def test_leading_digit(): + result = sanitize_to_python_id("2fast") + assert result == "name_2fast" + + +def test_mixed_special_digit_nonleading(): + result = sanitize_to_python_id("test-repo#1") + assert result == "test_repo_1" + + +def test_all_special_chars(): + result = sanitize_to_python_id("@#$%") + assert result == "____" diff --git a/tests/unit/test_services.py b/tests/unit/test_services.py index 230e385..88d7b26 100644 --- a/tests/unit/test_services.py +++ b/tests/unit/test_services.py @@ -14,14 +14,17 @@ def test_syncing_existing_repository() -> None: storage = adapters.FakeRepoStorage(location) storage.fake_is_cloned = True github_url = "https://github.com/programmer-ke/akash-docs-buddy.git" + branch = "trunk" # When we synchronise - services.sync_repository(github_url, storage) + services.sync_repository(github_url, branch, storage) # Then a pull is performed - assert len(storage.actions) == 1 - [(action,)] = storage.actions - assert action == "PULL" + assert len(storage.actions) == 2 + [(cmd0, arg0), (cmd1,)] = storage.actions + assert cmd0 == "CHECKOUT" + assert arg0 == branch + assert cmd1 == "PULL" def test_syncing_non_existent_repo_and_can_clone() -> None: @@ -31,16 +34,18 @@ def test_syncing_non_existent_repo_and_can_clone() -> None: storage.fake_is_cloned = False storage.fake_can_clone = True github_url = "https://github.com/programmer-ke/akash-docs-buddy.git" + branch = "trunk" # When we synchronise - services.sync_repository(github_url, storage) + services.sync_repository(github_url, branch, storage) # Then a clone is performed with the correct URL and target assert len(storage.actions) == 1 - [(action, url, target)] = storage.actions + [(action, url, target, actual_branch)] = storage.actions assert action == "CLONE" assert url == github_url assert target == location + assert branch == actual_branch def test_syncing_non_existent_repo_and_cannot_clone() -> None: @@ -50,11 +55,12 @@ def test_syncing_non_existent_repo_and_cannot_clone() -> None: storage.fake_is_cloned = False storage.fake_can_clone = False github_url = "https://github.com/programmer-ke/akash-docs-buddy.git" + branch = "master" # When we try to synchronise # Then a RepositorySyncError is raised with pytest.raises(services.RepositorySyncError): - services.sync_repository(github_url, storage) + services.sync_repository(github_url, branch, storage) # And no storage action was recorded assert len(storage.actions) == 0 @@ -282,7 +288,7 @@ def test_find_answer_with_configured_agent() -> None: services.index_document_chunks(pipeline, index) # And an agent has been configured with a search tool - tools = [adapters.make_search_tool(index)] + tools = [adapters.make_search_tool(index, "foo_docs", "Foo documentation")] research_user_query = adapters.make_fake_research_agent(prompt="some prompt") # When user submits valid query @@ -308,7 +314,7 @@ def test_agent_error_gracefully_handled() -> None: services.index_document_chunks(pipeline, index) # And an agent has been configured with a search tool - tools = [adapters.make_search_tool(index)] + tools = [adapters.make_search_tool(index, "foo_docs", "Foo documentation")] research_user_query = adapters.make_fake_research_agent(prompt="some prompt") # When user submits query likely to fail diff --git a/tests/unit/test_tools.py b/tests/unit/test_tools.py index 70db695..a4ed009 100644 --- a/tests/unit/test_tools.py +++ b/tests/unit/test_tools.py @@ -10,7 +10,17 @@ def test_can_create_search_tool_over_index() -> None: index = adapters.FakeIndex(pipeline) services.index_document_chunks(pipeline, index) - search_index_tool = adapters.make_search_tool(index) + repo_description = "The foo website documentation in markdown" + tool_id = "github_com_programmer_ke_docs_buddy" + search_index_tool = adapters.make_search_tool(index, tool_id, repo_description) + + # when we check description, it contains the repo information + tool_doc = search_index_tool.__doc__ + assert tool_doc and repo_description in tool_doc + + # when we check the tool name, it contains the tool id + assert tool_id in search_index_tool.__name__ + phrase = "provider" # When we search without a limit diff --git a/todo.md b/todo.md index e3c94aa..93f7661 100644 --- a/todo.md +++ b/todo.md @@ -10,10 +10,24 @@ Status legend: - [ ] address todo comments +- [ ] **US-008: User can submit a query via web form and receive a structured response** + *As a user, I want to submit a query via a web form and receive the structured response (answer + citations) rendered in the browser, so that I can use the system from a web interface.* + - [ ] **Scenario 8.1:** Web form query returns answer and citations + Given the documentation index has been built + And the AI agent is configured + When the user submits a query via the web form + Then the web page displays the answer and the list of citations + - [ ] **Scenario 8.2:** Web form query returns error message + Given the documentation index has not been built + When the user submits a query via the web form + Then the web page displays the error message: "Documentation index is not available. Please sync a repository first." + -- [ ] **US-007: User can submit a query via CLI and receive a structured response** +## in progress + +- [>] **US-007: User can submit a query via CLI and receive a structured response** *As a user, I want to submit a query via the CLI and receive the structured response (answer + citations) printed to the console, so that I can use the system from the command line.* - - [ ] **Scenario 7.1:** CLI query returns answer and citations + - [>] **Scenario 7.1:** CLI query returns answer and citations Given the documentation index has been built And the AI agent is configured When the user runs `docs-buddy query "What is X?"` @@ -32,22 +46,6 @@ Status legend: When the user runs `docs-buddy query "What is X?"` Then the CLI prints the error message: "Documentation index is not available. Please sync a repository first." -- [ ] **US-008: User can submit a query via web form and receive a structured response** - *As a user, I want to submit a query via a web form and receive the structured response (answer + citations) rendered in the browser, so that I can use the system from a web interface.* - - [ ] **Scenario 8.1:** Web form query returns answer and citations - Given the documentation index has been built - And the AI agent is configured - When the user submits a query via the web form - Then the web page displays the answer and the list of citations - - [ ] **Scenario 8.2:** Web form query returns error message - Given the documentation index has not been built - When the user submits a query via the web form - Then the web page displays the error message: "Documentation index is not available. Please sync a repository first." - - -## in progress - - ## done - [x] **US-006: Operator can observe logs**