feat: Implement CLI functionality for user query processing

.gitignore

@@ -11,3 +11,4 @@ __pycache__
 .whoosh
 
 bin/
+.docs_buddy.yaml

src/docs_buddy/_vendor/dotconfig.py

@@ -0,0 +1,75 @@
+"""dotconfig - Create config objects from JSON/YAML"""
+
+import json
+from pathlib import Path
+
+import yaml
+
+
+class Config:
+    """Wrapper for nested config with dot-notation access.
+
+    >>> config = Config({"database": {"host": "localhost", "port": 5432}})
+    >>> config.database.host
+    'localhost'
+    >>> config.database.port
+    5432
+
+    Supports loading JSON/YAML
+    """
+
+    def __new__(cls, config):
+        if isinstance(config, list):
+            return [cls(c) if isinstance(c, (list, dict)) else c for c in config]
+        return super().__new__(cls)
+
+    def __init__(self, config: dict):
+        self._config = config
+
+    @classmethod
+    def from_json(cls, file):
+        """Load from JSON file"""
+        data = cls._get_data(file, json.load)
+        return cls(data)
+
+    @classmethod
+    def from_yaml(cls, file):
+        """Load from YAML file"""
+        data = cls._get_data(file, yaml.safe_load)
+        return cls(data)
+
+    @staticmethod
+    def _get_data(source, loader):
+        """Load data from file object or path"""
+        try:
+            filepath = Path(source)
+        except TypeError:
+            # treat as filelike
+            data = loader(source)
+        else:
+            with open(filepath) as f:
+                data = loader(f)
+        return data
+
+    def __getattr__(self, name):
+        try:
+            value = self._config[name]
+        except KeyError as e:
+            raise AttributeError(f"No config '{name}' found.") from e
+
+        if isinstance(value, (dict, list)):
+            return Config(value)
+        return value
+
+    def __getitem__(self, name):
+        value = self._config[name]
+        if isinstance(value, (list, dict)):
+            return Config(value)
+        return value
+
+    def __str__(self):
+        return json.dumps(self._config)
+
+    def __repr__(self):
+        cls_name = type(self).__name__
+        return f"{cls_name}({self._config!r})"

src/docs_buddy/adapters/__init__.py

@@ -25,6 +25,10 @@ class MemoryBusError(DocsBuddyError):
     pass
 
 
+class FileSystemRepoStorageError(DocsBuddyError):
+    pass
+
+
 class InMemoryMessageBus:
     """In memory implementation of the message bus protocol"""
 
@@ -75,10 +79,11 @@ def is_already_cloned(self) -> bool:
     def can_clone(self) -> bool:
         return self.fake_can_clone
 
-    def clone_repo(self, url: str) -> None:
-        self.actions.append(("CLONE", url, self._target))
+    def clone_repo(self, url: str, branch: str) -> None:
+        self.actions.append(("CLONE", url, self._target, branch))
 
-    def pull_repo(self):
+    def pull_repo(self, branch: str) -> None:
+        self.actions.append(("CHECKOUT", branch))
         self.actions.append(("PULL",))
 
 
@@ -95,11 +100,19 @@ def __repr__(self):
     def is_already_cloned(self) -> bool:
         return self._is_git_repository(self._target)
 
-    def pull_repo(self):
-        self._update_repository(self._target)
+    def pull_repo(self, branch: str) -> None:
+        try:
+            self._update_repository(self._target, branch)
+        except subprocess.CalledProcessError as exc:
+            msg = f"Unable to pull branch '{branch}'. Error: '{exc.stderr}'"
+            raise FileSystemRepoStorageError(msg) from exc
 
-    def clone_repo(self, url: str) -> None:
-        self._clone_repository(url, self._target)
+    def clone_repo(self, url: str, branch: str) -> None:
+        try:
+            self._clone_repository(url, self._target, branch)
+        except subprocess.CalledProcessError as exc:
+            msg = f"Unable to clone repository '{url}'. Error: '{exc.stderr}'"
+            raise FileSystemRepoStorageError(msg) from exc
 
     def can_clone(self):
         """Indicates whether we can clone a new repo in the target
@@ -123,18 +136,37 @@ def _is_git_repository(directory: str) -> bool:
         return git_dir.exists() and git_dir.is_dir()
 
     @staticmethod
-    def _clone_repository(url: str, target_dir: str) -> None:
+    def _clone_repository(url: str, target_dir: str, branch: str) -> None:
         """Clone a git repository with depth 1."""
         subprocess.run(
-            ["git", "clone", "--depth", "1", url, target_dir],
+            ["git", "clone", "--depth", "1", "--branch", branch, url, target_dir],
             check=True,
             capture_output=True,
         )
 
     @staticmethod
-    def _update_repository(directory: str) -> None:
+    def _update_repository(directory: str, branch: str) -> None:
         """Pull latest changes from a git repository."""
-        subprocess.run(["git", "pull"], cwd=directory, check=True, capture_output=True)
+        subprocess.run(
+            [
+                "git",
+                "fetch",
+                "--depth",
+                "1",
+                "origin",
+                f"{branch}:refs/remotes/origin/{branch}",
+            ],
+            cwd=directory,
+            check=True,
+            capture_output=True,
+        )
+
+        subprocess.run(
+            ["git", "checkout", "-B", branch, f"origin/{branch}"],
+            cwd=directory,
+            capture_output=True,
+            check=True,
+        )
 
 
 class FakeIntermediateStorage:

src/docs_buddy/adapters/agent.py

@@ -80,8 +80,17 @@ async def on_tool_end(self, context, agent, tool, result):
         )
 
 
-def make_search_tool(index: services.DocumentIndex) -> Callable:
-    """Creates a search tool over the index"""
+def make_search_tool(
+    index: services.DocumentIndex, tool_id: str, content_information: str
+) -> Callable:
+    """Creates a search tool over the index
+
+    @arg index - the index to search over
+    @arg tool_id - the unique identifier used to create the tool name
+    @arg content_information - a description of content the tool searches
+
+    @returns - a search tool over the index
+    """
 
     @log_input(log, logging.INFO)
     def search_document_index(phrase: str, max_results: int = 5) -> list[str]:
@@ -95,6 +104,10 @@ def search_document_index(phrase: str, max_results: int = 5) -> list[str]:
           list[str]: A list of JSON formatted strings representing the
                      the results. Each result has associated content,
                      path and metadata fields
+
+        What is indexed:
+
+        {content_information}
         """
 
         if not max_results > 0:
@@ -106,6 +119,12 @@ def search_document_index(phrase: str, max_results: int = 5) -> list[str]:
         results = index.search(query, max_results)
         return [str(r) for r in results]
 
+    search_document_index.__doc__ = search_document_index.__doc__.format(
+        content_information=content_information
+    )
+
+    search_document_index.__name__ = search_document_index.__name__ + f"_{tool_id}"
+
     return search_document_index
 
 
@@ -152,28 +171,26 @@ def generate_final_response(
     return final_response
 
 
-def make_openai_research_agent(prompt: str) -> services.Agent:
+def make_openai_research_agent(
+    prompt: str, model_name: str, base_url: str
+) -> services.Agent:
     """Creates an openai agent"""
 
-    BASE_URL = os.getenv("OPENAI_BASE_URL") or ""
-    API_KEY = os.getenv("OPENAI_API_KEY") or ""
-    MODEL_NAME = os.getenv("DOCS_BUDDY_MODEL_NAME") or ""
+    api_key = os.getenv("OPENAI_API_KEY")
 
-    if not BASE_URL or not API_KEY or not MODEL_NAME:
-        raise AgentError(
-            "Please set OPENAI_BASE_URL, OPENAI_API_KEY, DOCS_BUDDY_MODEL_NAME"
-        )
+    if not api_key:
+        raise AgentError("Please set the OPENAI_API_KEY environment variable")
 
-    client = openai.AsyncOpenAI(base_url=BASE_URL, api_key=API_KEY)
+    client = openai.AsyncOpenAI(base_url=base_url, api_key=api_key)
     agents.set_tracing_disabled(disabled=True)
 
     class CustomModelProvider(agents.ModelProvider):
-        def get_model(self, model_name: str | None) -> agents.Model:
+        def get_model(self, name: str | None) -> agents.Model:
             return agents.OpenAIChatCompletionsModel(
-                model=model_name or MODEL_NAME, openai_client=client
+                model=name or model_name, openai_client=client
             )
 
-    CUSTOM_MODEL_PROVIDER = CustomModelProvider()
+    custom_model_provider = CustomModelProvider()
 
     def openai_agent(
         query: domain.Query, tools: list[Callable]
@@ -205,7 +222,7 @@ def openai_agent(
                 agent,
                 str(query),
                 LoggingHooks(),
-                agents.RunConfig(model_provider=CUSTOM_MODEL_PROVIDER),
+                agents.RunConfig(model_provider=custom_model_provider),
             )
         )
 

src/docs_buddy/adapters/whoosh_index.py

@@ -27,11 +27,14 @@ class WhooshDocumentIndex:
     )
     _SEARCH_FIELDS = ["content", "metadata", "path_keywords"]
 
-    def __init__(self, index_location: PathLike | None = None):
+    def __init__(
+        self, index_location: PathLike | None = None, file_prefix: str | None = None
+    ):
         """
         Initialize a Whoosh document index.
 
         """
+        self._file_prefix = file_prefix
         self._index = None
         if index_location:
             self._index = index.open_dir(index_location)
@@ -83,16 +86,26 @@ def search(self, query: domain.Query, max_results: int) -> list[domain.QueryResu
 
         parsed_query = self._query_parser.parse(str(query))
 
+        prefix = ""
+
+        if self._file_prefix:
+            prefix = (
+                self._file_prefix + "/"
+                if not self._file_prefix.endswith("/")
+                else self._file_prefix
+            )
+
         with self._index.searcher() as searcher:
             # todo: consider interaction between indexing and searching
             # is locking required for coordination?
             results = searcher.search(parsed_query, limit=max_results)
             results = [
                 domain.QueryResult(
                     content=r["content"],
-                    path=r["path"],
+                    path=prefix + r["path"],
                     metadata=json.loads(r["metadata"]),
                 )
                 for r in results
             ]
+
         return results

src/docs_buddy/common.py

@@ -8,6 +8,7 @@
 from typing import TypeAlias
 import logging
 import functools
+import re
 
 PathLike: TypeAlias = str | os.PathLike
 
@@ -45,3 +46,11 @@ def wrapper(*args, **kwargs):
         return wrapper
 
     return decorator
+
+
+def sanitize_to_python_id(identifier: str) -> str:
+    """Converts string to valid Python identifier"""
+    sanitized = re.sub(r"[^a-zA-Z0-9_]", "_", identifier)
+    if sanitized[0].isdigit():
+        sanitized = "name_" + sanitized
+    return sanitized

src/docs_buddy/entrypoints/bootstrap.py

@@ -17,6 +17,7 @@ def get_message_bus(
     repository_storage_path: PathLike,
     chunks_storage_path: PathLike,
     lexical_index_storage_path: PathLike,
+    doc_extensions: tuple[str, ...],
 ) -> handlers.MessageBus:
     """Create and configure the application message bus with all dependencies.
 
@@ -31,7 +32,7 @@ def get_message_bus(
     message_bus = adapters.InMemoryMessageBus()
     repo_storage = adapters.FileSystemRepoStorage(repository_storage_path)
     docs_storage = adapters.FileSystemDocsStorage(
-        repository_storage_path, chunks_storage_path
+        repository_storage_path, chunks_storage_path, doc_extensions
     )
     chunks_pipeline = adapters.FileSystemDocumentChunksPipeline(
         chunks_storage_path, lexical_index_storage_path

src/docs_buddy/entrypoints/cli/README.md

@@ -6,15 +6,20 @@ from the terminal.
 ## Usage
 
 ```bash
-usage: python -m docs_buddy.entrypoints.cli [-h] [--repo-id REPO_IDS] [--update-sources | query]
+usage: __main__.py [-h] [--log-level {DEBUG,INFO,WARNING,ERROR,CRITICAL}] [--update-sources] [--init]
+                   [--current-config]
+                   [query]
 
 Docs Buddy CLI
 
 positional arguments:
-  query               Query string to search the documentation
+  query                 Query string to search the documentation
 
 options:
-  -h, --help          show this help message and exit
-  --repo-id REPO_IDS  Repository ID(s). Can be used multiple times, each time for a different ID
-  --update-sources    Update document sources from repository and recreate index
+  -h, --help            show this help message and exit
+  --log-level {DEBUG,INFO,WARNING,ERROR,CRITICAL}
+                        The log level to use (default WARNING)
+  --update-sources      Update document sources from repository and recreate index
+  --init                Create a .docs_buddy.yaml config file from template in the current directory
+  --current-config      Show location of the current config file in use
 ```