Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/e2e-gpu-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -79,4 +79,4 @@ jobs:
run: mise run --no-deps --skip-deps cluster

- name: Run tests
run: mise run --no-deps --skip-deps e2e:python:gpu
run: mise run --no-deps --skip-deps e2e:k3s:gpu
4 changes: 2 additions & 2 deletions .github/workflows/e2e-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@ jobs:
matrix:
include:
- suite: python
cmd: "mise run --no-deps --skip-deps e2e:python"
cmd: "mise run --no-deps --skip-deps e2e:docker:python"
- suite: rust
cmd: "mise run --no-deps --skip-deps e2e:rust"
cmd: "mise run --no-deps --skip-deps e2e:docker:rust"
container:
image: ghcr.io/nvidia/openshell/ci:latest
credentials:
Expand Down
32 changes: 30 additions & 2 deletions TESTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@

```bash
mise run test # Rust + Python unit tests
mise run e2e # End-to-end tests (starts a Docker-backed gateway)
mise run e2e # Docker-backed end-to-end tests
mise run e2e:docker # Same as e2e, explicit Docker-backed suite
mise run e2e:k3s:gpu # GPU E2E against the OpenShell k3s-cluster gateway
mise run ci # Everything: lint, compile checks, and tests
```

Expand Down Expand Up @@ -96,6 +98,12 @@ def test_exec_returns_stdout(sandbox):
assert "hello" in result.stdout
```

Run Docker-backed Python e2e tests:

```bash
mise run e2e:docker:python
```

#### `Sandbox.exec_python`

`exec_python` serializes a Python callable with `cloudpickle`, sends it to the
Expand Down Expand Up @@ -154,7 +162,7 @@ Tests:
Run all CLI e2e tests:

```bash
mise run e2e:rust
mise run e2e:docker:rust
```

Run a single test directly with cargo:
Expand All @@ -163,6 +171,25 @@ Run a single test directly with cargo:
cargo test --manifest-path e2e/rust/Cargo.toml --features e2e --test sync
```

Run a single Rust e2e test target with the Docker-backed gateway wrapper:

```bash
OPENSHELL_E2E_RUST_TEST=smoke mise run e2e:docker:rust
OPENSHELL_E2E_RUST_TEST=sync mise run e2e:docker:rust
```

Run only the Docker-backed smoke test:

```bash
mise run e2e:docker:smoke
```

Run GPU E2E against the OpenShell k3s-cluster gateway:

```bash
mise run e2e:k3s:gpu
```

The harness (`e2e/rust/src/harness/`) provides:

| Module | Purpose |
Expand All @@ -178,3 +205,4 @@ The harness (`e2e/rust/src/harness/`) provides:
|---|---|
| `OPENSHELL_GATEWAY` | Override active gateway name for E2E tests |
| `OPENSHELL_GATEWAY_ENDPOINT` | Run E2E tests against an existing plaintext HTTP gateway endpoint |
| `OPENSHELL_E2E_RUST_TEST` | Run one Rust E2E integration test target, for example `smoke` or `sync` |
15 changes: 5 additions & 10 deletions e2e/rust/e2e-docker.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,14 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

# Run a Rust e2e test against a standalone gateway running the bundled Docker
# compute driver. Set OPENSHELL_GATEWAY_ENDPOINT=http://host:port to reuse an
# existing plaintext gateway instead of starting an ephemeral one.
# Backward-compatible wrapper for the Docker Rust e2e smoke path.
#
# Prefer e2e-rust.sh with OPENSHELL_E2E_RUST_TEST for new task wiring.

set -euo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
E2E_TEST="${OPENSHELL_E2E_DOCKER_TEST:-smoke}"

cargo build -p openshell-cli --features openshell-core/dev-settings
export OPENSHELL_E2E_RUST_TEST="${OPENSHELL_E2E_RUST_TEST:-${OPENSHELL_E2E_DOCKER_TEST:-smoke}}"

exec "${ROOT}/e2e/with-docker-gateway.sh" \
cargo test --manifest-path "${ROOT}/e2e/rust/Cargo.toml" \
--features e2e \
--test "${E2E_TEST}" \
-- --nocapture
exec "${ROOT}/e2e/rust/e2e-rust.sh"
31 changes: 31 additions & 0 deletions e2e/rust/e2e-rust.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

# Run Rust e2e tests against a standalone gateway running the bundled Docker
# compute driver. Set OPENSHELL_E2E_RUST_TEST to a Rust integration test target
# such as "smoke" or "sync" to run only that file.

set -euo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
E2E_TEST="${OPENSHELL_E2E_RUST_TEST:-}"

cargo build -p openshell-cli --features openshell-core/dev-settings

cargo_args=(
test
--manifest-path "${ROOT}/e2e/rust/Cargo.toml"
--features e2e
)

test_args=()

if [ -n "${E2E_TEST}" ]; then
cargo_args+=(--test "${E2E_TEST}")
test_args+=(--nocapture)
else
test_args+=(--skip docker_gpu_sandbox_runs_nvidia_smi)
fi

exec "${ROOT}/e2e/with-docker-gateway.sh" cargo "${cargo_args[@]}" -- "${test_args[@]}"
1 change: 1 addition & 0 deletions mise.lock
Original file line number Diff line number Diff line change
Expand Up @@ -300,6 +300,7 @@ url = "https://storage.googleapis.com/skaffold/releases/v2.19.0/skaffold-linux-a
url = "https://storage.googleapis.com/skaffold/releases/v2.19.0/skaffold-linux-amd64"

[tools.skaffold."platforms.macos-arm64"]
checksum = "blake3:0e91ba4f1d53adfdf70868424b3678de1b4e194488652ba5ed1805eb7142f929"
url = "https://storage.googleapis.com/skaffold/releases/v2.19.0/skaffold-darwin-arm64"

[tools.skaffold."platforms.macos-x64"]
Expand Down
60 changes: 45 additions & 15 deletions tasks/test.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,16 @@ description = "Run all tests (Rust + Python)"
depends = ["test:rust", "test:python"]

[e2e]
description = "Run all end-to-end tests (Rust + Python)"
depends = ["e2e:rust", "e2e:python"]
description = "Run all Docker-backed end-to-end tests (Rust + Python)"
depends = ["e2e:docker"]

["e2e:gpu"]
description = "Run GPU end-to-end tests"
depends = ["e2e:python:gpu"]
description = "Alias for e2e:k3s:gpu"
run = [
"echo 'warning: e2e:gpu is deprecated; use e2e:k3s:gpu' >&2",
"mise run e2e:k3s:gpu",
]
hide = true

["test:rust"]
description = "Run Rust tests"
Expand All @@ -28,20 +32,31 @@ run = "uv run pytest python/"
hide = true

["e2e:rust"]
description = "Run Rust CLI e2e tests against a Docker-backed gateway"
description = "Alias for e2e:docker:rust"
run = [
"cargo build -p openshell-cli --features openshell-core/dev-settings",
"e2e/with-docker-gateway.sh cargo test --manifest-path e2e/rust/Cargo.toml --features e2e -- --skip docker_gpu_sandbox_runs_nvidia_smi",
"echo 'warning: e2e:rust is deprecated; use e2e:docker:rust' >&2",
"mise run e2e:docker:rust",
]
hide = true

["e2e:python"]
description = "Run Python e2e tests against a Docker-backed gateway (E2E_PARALLEL=N or 'auto'; default 5)"
depends = ["python:proto"]
env = { UV_NO_SYNC = "1", PYTHONPATH = "python" }
run = "e2e/with-docker-gateway.sh uv run pytest -o python_files='test_*.py' -m 'not gpu' -n ${E2E_PARALLEL:-5} e2e/python"
description = "Alias for e2e:docker:python"
run = [
"echo 'warning: e2e:python is deprecated; use e2e:docker:python' >&2",
"mise run e2e:docker:python",
]
hide = true

["e2e:python:gpu"]
description = "Run Python GPU e2e tests"
description = "Alias for e2e:k3s:gpu"
run = [
"echo 'warning: e2e:python:gpu is deprecated; use e2e:k3s:gpu' >&2",
"mise run e2e:k3s:gpu",
]
hide = true

["e2e:k3s:gpu"]
Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@drew does e2e:k3s:gpu feel like a reasonable name for the GPU E2E task that runs through the k3s-cluster gateway, or would you prefer a different name?

description = "Run GPU e2e tests against an OpenShell k3s-cluster gateway"
depends = ["python:proto", "CLUSTER_GPU=1 cluster"]
env = { UV_NO_SYNC = "1", PYTHONPATH = "python" }
run = "uv run pytest -o python_files='test_*.py' -m gpu -n ${E2E_PARALLEL:-1} e2e/python"
Expand All @@ -56,10 +71,25 @@ description = "Start openshell-gateway with the VM compute driver and run the cl
run = "e2e/rust/e2e-vm.sh"

["e2e:docker"]
description = "Run all Docker-backed end-to-end tests (Rust + Python)"
depends = ["e2e:docker:rust", "e2e:docker:python"]

["e2e:docker:rust"]
description = "Run Rust CLI e2e tests against a Docker-backed gateway"
run = "e2e/rust/e2e-rust.sh"

["e2e:docker:python"]
description = "Run Python e2e tests against a Docker-backed gateway (E2E_PARALLEL=N or 'auto'; default 5)"
depends = ["python:proto"]
env = { UV_NO_SYNC = "1", PYTHONPATH = "python" }
run = "e2e/with-docker-gateway.sh uv run pytest -o python_files='test_*.py' -m 'not gpu' -n ${E2E_PARALLEL:-5} e2e/python"

["e2e:docker:smoke"]
description = "Run smoke e2e against a standalone gateway with the Docker compute driver"
run = "e2e/rust/e2e-docker.sh"
env = { OPENSHELL_E2E_RUST_TEST = "smoke" }
run = "e2e/rust/e2e-rust.sh"

["e2e:docker:gpu"]
description = "Run GPU e2e against a standalone gateway with the Docker compute driver"
env = { OPENSHELL_E2E_DOCKER_GPU = "1", OPENSHELL_E2E_DOCKER_TEST = "docker_gpu" }
run = "e2e/rust/e2e-docker.sh"
env = { OPENSHELL_E2E_DOCKER_GPU = "1", OPENSHELL_E2E_RUST_TEST = "docker_gpu" }
run = "e2e/rust/e2e-rust.sh"
Loading