diff --git a/README.md b/README.md index 8534fed..b3f9a74 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -![AVISE logo](/docs/assets/avise_logo.png) +![](/docs/assets/avise_logo.png) # AVISE - AI Vulnerability Identification & Security Evaluation @@ -15,52 +15,78 @@ A framework for identifying vulnerabilities in and evaluating the security of AI ### Prerequisites - Python 3.10+ -- Docker (for running models backend) -- pip +- Docker (For Running models locally with Ollama) -### 1. Clone the Repository +### 1. Install AVISE + +Install with +- **pip:** + ```bash + pip install avise + ``` + +- **uv:** + + ```bash + uv install avise + ``` + +### 2. Run a model + +You can use AVISE to evaluate any model accessible via an API by configuring a Connector. In this Quickstart, we will +assume using the Ollama Docker container for running a language model. If you wish to evaluate models deployed in other ways, see +the [Full Documentations](https://avise.readthedocs.io) and available template connector configuration files at `AVISE/avise/configs/connector/languagemodel/` dir of this repository. + +#### Running a language model locally with Docker & Ollama + +- Clone this repository to your local machine with: ```bash git clone https://github.com/ouspg/AVISE.git -cd AVISE ``` -### 2. Set Up Python Environment +- Create the Ollama Docker container + - for **GPU** accelerated inference with: + ```bash + docker compose -f AVISE/docker/ollama/docker-compose.yml up -d + ``` + - or for **CPU** inference with: + ```bash + docker compose -f AVISE/docker/ollama/docker-compose-cpu.yml up -d + ``` -```bash -# Create virtual environment -python -m venv venv +- Pull an Ollama model to evaluate into the container with: + ```bash + docker exec -it avise-ollama ollama pull + ``` + +### 3. Evaluate the model with a Security Evaluation Test (SET) -source venv/bin/activate # Or venv/Scripts/Activate on Windows +#### Basic usage -# Install dependencies -pip install -r requirements.txt +```bash +avise --SET --connectorconf [options] ``` -### 3. Set Up by using Ollama Backend with Docker +For example, you can run the `prompt_injection` SET on the model pulled to the Ollama Docker container with: -**GPU Version:** ```bash -docker-compose -f docker/ollama/docker-compose.yml up -d +avise --SET prompt_injection --connectorconf ollama_lm --target ``` -**CPU-only Version:** +To list the available SETs, run the command: ```bash -docker-compose -f docker/ollama/docker-compose-cpu.yml up -d +avise --SET-list ``` -### 4. Pull Models -After Ollama is running, pull the models you want to test: +## Advanced usage -```bash -# Pull models for testing and for evaluation -docker exec -it avise-ollama ollama pull -``` +### Configuring Connectors -### 5. Configure Connectors +You can create your own connector configuration files, or if you cloned the AVISE repository, you can modify the existing connector configuration files in `AVISE/avise/configs/connector/languagemodel/`. -Edit `avise/configs/connector/languagemodel/ollama.json`: +For example, you can edit the default Ollama Connector configuration file `AVISE/avise/configs/connector/languagemodel/ollama.json`, and insert the name of an Ollama model you have pulled to be used as a target by default: ```json { @@ -73,27 +99,10 @@ Edit `avise/configs/connector/languagemodel/ollama.json`: } } ``` - -## Usage - -### Basic usage - -```bash -python -m avise --SET --connectorconf [options] -``` - -For example, you can run the `prompt_injection` Security Evaluation Test on a target model running locally via Ollama with: - -```bash -python -m avise --SET prompt_injection --connectorconf ollama_lm -``` - -### Advanced usage - If you want to use custom configuration files for SETs and/or Connectors, you can do so by giving the paths to the configuration files with `--SETconf` and `--connectorconf` arguments: ```bash -python -m avise --SET prompt_injection --SETconf avise/configs/SET/languagemodel/single_turn/prompt_injection_mini.json --connectorconf avise/configs/connector/languagemodel/ollama.json +avise --SET prompt_injection --SETconf AVISE/avise/configs/SET/languagemodel/single_turn/prompt_injection_mini.json --connectorconf AVISE/avise/configs/connector/languagemodel/ollama.json ``` ### Required Arguments @@ -109,11 +118,14 @@ python -m avise --SET prompt_injection --SETconf avise/configs/SET/languagemodel | Argument | Description | |----------|-------------| | `--SETconf` | Path to SET configuration JSON file. If not given, uses preconfigured paths for SET config JSON files. | +| `--target`, `-t` | Name of the target model/system to evaluate. Overrides target name from connector configuration file. | | `--format`, `-f` | Report format: `json`, `html`, `md` | | `--runs`, `-r` | How many times each SET is executed | | `--output` | Custom output file path | -| `--reports-dir` | Base directory for reports (default: `reports/`) | -| `--SET_list` | List available Security Evaluation Tests | -| `--connector_list` | List available Connectors | +| `--reports-dir` | Base directory for reports (default: `avise-reports/`) | +| `--SET-list` | List available Security Evaluation Tests | +| `--connector-list` | List available Connectors | | `--verbose`, `-v` | Enable verbose logging | | `--version`, `-V` | Print version | + + diff --git a/avise/cli.py b/avise/cli.py index 2820e3e..bda784a 100644 --- a/avise/cli.py +++ b/avise/cli.py @@ -61,12 +61,12 @@ def main(arguments=None) -> None: description="AVISE - AI Vulnerability Identification & Security Evaluation" ) parser.add_argument( - "--SET_list", + "--SET-list", action="store_true", help="List available Security Evaluation Tests", ) parser.add_argument( - "--connector_list", + "--connector-list", action="store_true", help="List available connectors and formats", ) @@ -87,6 +87,10 @@ def main(arguments=None) -> None: "--SETconf", help="Path to Security Evaluation Test configuration JSON" ) + parser.add_argument( + "--target", "-t", help="Name of the target model or system to evaluate" + ) + parser.add_argument( "--elm", help="Boolean indicator whether to use an Evaluation Language Model to evaluate SET results or not. True or False. Default: True", @@ -112,7 +116,7 @@ def main(arguments=None) -> None: help="How many times each SET is executed (default 1).", ) parser.add_argument( - "--reports_dir", + "--reports-dir", "-d", default=DEFAULT_REPORTS_DIR, help=f"Base directory for reports (default: {DEFAULT_REPORTS_DIR}).", @@ -210,11 +214,12 @@ def main(arguments=None) -> None: set_config_path=set_config_path, connector_config_path=args.connectorconf, evaluation_model_name=args.elm, - output_path=args.output, report_format=report_format, reports_dir=args.reports_dir, generate_ai_summary=args.ai_summary, runs=args.runs, + output_path=args.output, + target=args.target, ) # Print a small summary to the console diff --git a/avise/engine.py b/avise/engine.py index 6e1fcdd..e0dceac 100644 --- a/avise/engine.py +++ b/avise/engine.py @@ -94,11 +94,12 @@ def run_test( set_config_path: str, connector_config_path: str, evaluation_model_name: str, - output_path: Optional[str] = None, report_format: ReportFormat = ReportFormat.HTML, reports_dir: str = DEFAULT_REPORTS_DIR, generate_ai_summary: bool = True, runs: int = 1, + output_path: Optional[str] = None, + target: str = Optional[None], ) -> dict: """Run the 4-phase pipeline @@ -117,6 +118,12 @@ def run_test( """ # Load model configuration connector_config = self.load_connector_config(connector_config_path) + # If provided with `target`, override target model from configuration file with it + if target is not None: + if "name" in connector_config["target_model"]: + connector_config["target_model"]["name"] = target + # TODO: Once there are default connectors for other system/model types than language models, + # add logic here to replace possible "name" in their config files with `target`. # Create a connector for the target model connector = self._build_connector(connector_config, evaluation=False)