From e1fff995a289bc0d21bd81fd500ddfaf95b042a4 Mon Sep 17 00:00:00 2001 From: Philippe Martin Date: Tue, 1 Sep 2026 15:44:53 +0200 Subject: [PATCH 1/3] feat(vm): support corporate HTTP forward proxy egress for microVM sandboxes The corporate forward proxy machinery from #1792 is driver-agnostic and already merged: openshell-supervisor-network implements CONNECT chaining, NO_PROXY matching, credentials, https:// proxies and corporate CA trust, and openshell-sandbox exposes it as six argv-only flags. Podman gained the driver half in #2245/#2512 and Kubernetes in #2633; the VM driver had none of it, so VM sandboxes on proxy-only networks could not reach any destination requiring the proxy even when policy allowed it. The blocking piece was not proxy logic but delivery: the VM guest init script runs as PID 1 and execs a fixed supervisor command line, and libkrun's krun_set_exec receives an empty argv, so there was no channel for driver-owned supervisor arguments. The supervisor's proxy flags deliberately have no environment fallback, and build_guest_environment merges user-supplied environment, so the guest env is not a safe transport either. Add a driver-authored argument file, mirroring the existing init.d manifest: the driver writes /opt/openshell/supervisor-args into the overlay upperdir on every launch and the guest reads it verbatim, one argument per line, appending it to every supervisor exec. It is written even when empty, which is what makes the channel unforgeable -- the upperdir always shadows the read-only image layer, so an image can neither supply its own arguments nor disable the operator's by omitting the file. Because both launch backends exec the same init script, this covers libkrun and QEMU without touching either. A microVM has no bind mounts or container secrets, so the credential and CA bundle are staged into the per-sandbox overlay the way the gateway JWT already is: credential root-only at 0600, CA at 0644, both rewritten every launch so a removed setting clears prior material, and both deleted with the sandbox state directory. This places the credential at rest in the overlay image on the gateway host, which differs from the Podman secret model and is documented as an explicit security consideration. Validation is fail-closed and shared: a new openshell_core::driver_utils::validate_upstream_proxy_settings holds the pairing rules the Podman driver established, and both the gateway and the driver call it so an invalid table names the offending key instead of surfacing as an opaque driver-readiness timeout. Guest egress leaves through gvproxy, so a proxy on the gateway host's loopback is reachable only through host.openshell.internal; the guest to gateway callback is unaffected. Closes #3088 Signed-off-by: Philippe Martin --- .../skills/debug-openshell-cluster/SKILL.md | 43 + architecture/sandbox.md | 24 + crates/openshell-core/src/container_paths.rs | 29 + crates/openshell-core/src/driver_utils.rs | 309 ++++++ crates/openshell-driver-vm/README.md | 8 + .../scripts/openshell-vm-sandbox-init.sh | 73 +- crates/openshell-driver-vm/src/driver.rs | 668 ++++++++++++- crates/openshell-driver-vm/src/main.rs | 87 ++ crates/openshell-gateway/src/vm.rs | 176 +++- docs/reference/gateway-config.mdx | 37 + docs/reference/sandbox-compute-drivers.mdx | 10 + e2e/rust/Cargo.toml | 5 + e2e/rust/e2e-vm.sh | 1 + e2e/rust/src/harness/host_process.rs | 103 ++ e2e/rust/src/harness/mod.rs | 1 + e2e/rust/tests/vm_corporate_proxy.rs | 885 ++++++++++++++++++ 16 files changed, 2451 insertions(+), 8 deletions(-) create mode 100644 e2e/rust/src/harness/host_process.rs create mode 100644 e2e/rust/tests/vm_corporate_proxy.rs diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index 47588b1aeb..ee542f17dc 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -630,6 +630,49 @@ openshell status openshell logs ``` +#### Corporate upstream proxy + +When VM sandbox egress routes through a corporate HTTP forward proxy, the +operator-owned settings live under `[openshell.drivers.vm]` and the gateway +forwards them to the `openshell-driver-vm` subprocess as `--https-proxy`, +`--no-proxy`, `--proxy-auth-file`, `--proxy-auth-allow-insecure`, +`--proxy-connect-by-hostname`, and `--proxy-ca-bundle`. Both the gateway and +the driver validate them at startup, so any present-but-invalid value fails +closed with an error naming the key rather than reverting to a direct dial. +Confirm the configuration and the resulting driver argv first: + +```bash +grep -A20 '^\[openshell.drivers.vm\]' | grep -E 'https_proxy|no_proxy|proxy_auth_file|proxy_auth_allow_insecure|proxy_connect_by_hostname|proxy_ca_bundle' +ps -o args= -p "$(pgrep -f openshell-driver-vm | head -n1)" | tr ' ' '\n' | grep -A1 -- '--proxy\|--https-proxy\|--no-proxy' +``` + +Reachability is the most common failure. Guest egress leaves through gvproxy, +so a proxy bound to the gateway host's loopback is **not** reachable at +`127.0.0.1` from inside the guest: it must be addressed as +`http://host.openshell.internal:`, which gvproxy NATs from +`192.168.127.254` to the host's `127.0.0.1`. A `https_proxy` pointing at a +loopback URL produces policy-approved CONNECT attempts that time out while +public destinations still work. + +The settings reach the supervisor through a driver-written argument file in +the per-sandbox overlay, not through the guest environment. The credential and +CA bundle are staged into the same overlay at fixed guest paths. Inspect the +guest side from the VM console log, which records how many driver-supplied +arguments the init script read: + +```bash +grep -E 'supervisor arguments from driver|supervisor argument list' /sandboxes//rootfs-console.log +grep -Ei 'upstream|connect|proxy' /sandboxes//rootfs-console.log | tail -n 40 +``` + +`FATAL: supervisor argument list ... is not readable` or `FATAL: empty entry in +supervisor argument list` means the overlay is broken or was tampered with, and +the guest deliberately aborts rather than starting a supervisor with a +truncated egress configuration. If the guest logs no driver arguments at all +while `gateway.toml` sets `https_proxy`, the running driver predates the +configuration — check that the gateway spawned the driver binary you expect +(`[openshell.drivers.vm].driver_dir`). + ## Common Failure Patterns | Symptom | Likely cause | Check | diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 6e4e020536..0a169ad363 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -300,6 +300,30 @@ file and builds the `Proxy-Authorization: Basic` header; a credential that is empty, contains control characters, or is not in `user:pass` form is fatal on both sides. +The VM driver has no argv seam of its own: its guest init script runs as PID 1 +and execs a fixed supervisor command line, and the libkrun and QEMU launch +backends both reach the supervisor through that script. Driver-owned +supervisor arguments therefore travel in a per-sandbox file the driver writes +into the overlay upperdir at a fixed guest path, one argument per line, which +the guest reads verbatim (no word splitting or globbing) and appends to every +supervisor exec. The file is written on **every** launch, including an empty +file when there is nothing to pass: the upperdir copy always shadows the +read-only image layer, so a sandbox image can neither supply its own +supervisor arguments by baking a file at that path nor disable the operator's +by omitting one. This mirrors the driver-authored `init.d` manifest, which +solves the same trust problem for guest init drop-ins. + +A microVM has no bind mounts or container secrets, so the VM driver stages the +credential and the CA bundle into the per-sandbox overlay disk instead — the +credential root-only, the CA world-readable, both at fixed `/opt/openshell` +paths and both removed with the sandbox state directory. The consequence, +which differs from the Podman secret model, is that the credential is at rest +inside that overlay image on the gateway host; the per-sandbox gateway JWT +already travels the same path. Guest egress leaves through gvproxy, so a proxy +on the gateway host's loopback is reachable only through the host alias +`host.openshell.internal`; the guest's gateway callback is unaffected and +never traverses the proxy. + For Kubernetes sandboxes, the operator configures a Secret name and key rather than a gateway-host file path. Kubernetes projects that Secret only into the container that runs network supervision. Proxy credential Secrets require the diff --git a/crates/openshell-core/src/container_paths.rs b/crates/openshell-core/src/container_paths.rs index c63e4bcdd8..e26ea53f7a 100644 --- a/crates/openshell-core/src/container_paths.rs +++ b/crates/openshell-core/src/container_paths.rs @@ -65,6 +65,32 @@ pub const VM_GUEST_TLS_KEY_PATH: &str = "/opt/openshell/tls/tls.key"; pub const VM_GUEST_SANDBOX_TOKEN_PATH: &str = "/opt/openshell/auth/sandbox.jwt"; pub const VM_GUEST_INIT_DROPIN_DIR: &str = "/opt/openshell/init.d"; pub const VM_GUEST_INIT_DROPIN_MANIFEST: &str = "/opt/openshell/init.d.manifest"; + +/// Guest path for the corporate upstream-proxy credential in VM sandboxes. +/// +/// The VM driver stages the `user:pass` credential here (mode `0600`, +/// root-only) inside the per-sandbox overlay upperdir, and passes only this +/// path on the supervisor's argv. A microVM has no bind mounts or container +/// secrets, so this is the same delivery the per-sandbox JWT already uses. +pub const VM_GUEST_UPSTREAM_PROXY_AUTH_PATH: &str = "/opt/openshell/auth/upstream-proxy"; + +/// Guest path for the corporate proxy CA bundle in VM sandboxes. +/// +/// A CA certificate is not secret, so unlike the credential this is staged +/// world-readable. The supervisor trusts it for the handshake with an +/// `https://` proxy and for server certificates re-signed by a +/// TLS-intercepting proxy. +pub const VM_GUEST_PROXY_CA_PATH: &str = "/opt/openshell/tls/proxy-ca.pem"; + +/// Guest path for the driver-authored supervisor argument list in VM sandboxes. +/// +/// Podman and Kubernetes build the supervisor's command line directly; the VM +/// guest init script execs a fixed argv, so driver-owned arguments travel +/// through this file instead. The driver writes it into the overlay upperdir +/// on every launch — empty when it has no arguments to pass — so a sandbox +/// image can neither forge entries nor shadow the driver's copy, and the +/// guest appends exactly what it finds there and nothing else. +pub const VM_GUEST_SUPERVISOR_ARGS_PATH: &str = "/opt/openshell/supervisor-args"; pub const VM_UMOCI_PATH: &str = "/opt/openshell/bin/umoci"; pub const VM_SANDBOX_OWNER_NORMALIZED_MARKER: &str = "/opt/openshell/.sandbox-owner-normalized"; @@ -103,6 +129,9 @@ mod tests { VM_GUEST_SANDBOX_TOKEN_PATH, VM_GUEST_INIT_DROPIN_DIR, VM_GUEST_INIT_DROPIN_MANIFEST, + VM_GUEST_UPSTREAM_PROXY_AUTH_PATH, + VM_GUEST_PROXY_CA_PATH, + VM_GUEST_SUPERVISOR_ARGS_PATH, VM_UMOCI_PATH, VM_SANDBOX_OWNER_NORMALIZED_MARKER, ]; diff --git a/crates/openshell-core/src/driver_utils.rs b/crates/openshell-core/src/driver_utils.rs index c8be114ebd..2e90a7d1a2 100644 --- a/crates/openshell-core/src/driver_utils.rs +++ b/crates/openshell-core/src/driver_utils.rs @@ -430,6 +430,126 @@ pub fn read_upstream_proxy_credential_file(path: &str) -> Result Ok(buf) } +/// Operator-supplied corporate upstream-proxy settings, as a borrowed view. +/// +/// Compute drivers store these keys under their own +/// `[openshell.drivers.]` table; this type exists so the pairing rules +/// between them live in one place instead of being restated per driver. +/// Field names map 1:1 onto the documented TOML keys `https_proxy`, +/// `no_proxy`, `proxy_auth_file`, `proxy_auth_allow_insecure`, +/// `proxy_connect_by_hostname`, and `proxy_ca_bundle`. +#[derive(Debug, Clone, Copy, Default)] +pub struct UpstreamProxySettings<'a> { + /// `https_proxy`: the corporate forward proxy URL. + pub url: Option<&'a str>, + /// `no_proxy`: comma-separated bypass list. + pub no_proxy: Option<&'a str>, + /// `proxy_auth_file`: host path to a `user:pass` credential file. + pub auth_file: Option<&'a str>, + /// `proxy_auth_allow_insecure`: acknowledgement that Basic auth to an + /// `http://` proxy travels in cleartext. + pub auth_allow_insecure: Option, + /// `proxy_connect_by_hostname`: send hostnames rather than validated IPs + /// in CONNECT requests. + pub connect_by_hostname: Option, + /// `proxy_ca_bundle`: host path to a PEM CA bundle trusted for the proxy. + pub ca_bundle: Option<&'a str>, +} + +/// Validate operator-supplied corporate upstream-proxy settings, fail-closed. +/// +/// Shares URL semantics with the in-container supervisor through +/// [`parse_upstream_proxy_url`], so a value accepted here can never be +/// rejected by the supervisor at sandbox startup (or vice versa). Every +/// auxiliary setting is only meaningful relative to a proxy boundary the +/// operator believed was in effect, so a stray one is rejected rather than +/// silently accepted while all egress dials directly. +/// +/// A present-but-empty string is rejected everywhere: the supervisor treats +/// an empty driver-supplied argument as a fatal misconfiguration, so a driver +/// must never accept (and later pass) one. +/// +/// # Errors +/// +/// Returns a message naming the offending key. +pub fn validate_upstream_proxy_settings( + settings: &UpstreamProxySettings<'_>, +) -> Result<(), String> { + let proxy_secure = if let Some(url) = settings.url { + let addr = parse_upstream_proxy_url(url).map_err(|err| match err { + UpstreamProxyUrlError::Empty => "https_proxy must not be empty when set".to_string(), + UpstreamProxyUrlError::InlineCredentials => { + "https_proxy must not embed credentials in the URL; supply them via \ + proxy_auth_file so they are not stored in config or sandbox metadata" + .to_string() + } + err => format!("https_proxy {err}"), + })?; + addr.secure + } else { + false + }; + + if let Some(list) = settings.no_proxy { + if list.trim().is_empty() { + return Err("no_proxy must not be empty when set; omit it instead".to_string()); + } + if settings.url.is_none() { + return Err("no_proxy is set but no https_proxy is configured".to_string()); + } + } + + if let Some(path) = settings.auth_file { + if path.trim().is_empty() { + return Err("proxy_auth_file must not be empty when set".to_string()); + } + if settings.url.is_none() { + return Err("proxy_auth_file is set but no https_proxy is configured".to_string()); + } + // Basic auth over the plain-TCP proxy connection is readable by + // anyone on the network path; sending it requires an explicit + // operator acknowledgement rather than being an implicit side effect + // of configuring credentials. For an https:// proxy the credential is + // inside the verified TLS session, so the acknowledgement is + // unnecessary (but tolerated). + if settings.auth_allow_insecure != Some(true) && !proxy_secure { + return Err( + "proxy_auth_file sends the credential as cleartext Basic auth over the \ + plain-TCP connection to the http:// proxy; set proxy_auth_allow_insecure \ + = true to accept that exposure, or remove proxy_auth_file" + .to_string(), + ); + } + } else if settings.auth_allow_insecure.is_some() { + // The acknowledgement without credentials means the operator believed + // an auth file was configured; surface the mismatch. + return Err( + "proxy_auth_allow_insecure is set but no proxy_auth_file is configured".to_string(), + ); + } + + if settings.connect_by_hostname.is_some() && settings.url.is_none() { + return Err( + "proxy_connect_by_hostname is set but no https_proxy is configured".to_string(), + ); + } + + // A CA bundle only makes sense relative to a proxy boundary (an https:// + // proxy handshake, or a TLS-intercepting proxy's re-sign CA). The file's + // readability and certificate content are checked at sandbox-create time + // by the driver and fail closed again in the supervisor. + if let Some(path) = settings.ca_bundle { + if path.trim().is_empty() { + return Err("proxy_ca_bundle must not be empty when set".to_string()); + } + if settings.url.is_none() { + return Err("proxy_ca_bundle is set but no https_proxy is configured".to_string()); + } + } + + Ok(()) +} + /// Container-side directory where the provider SPIFFE Workload API socket is mounted. pub const PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR: &str = "/spiffe-workload-api"; @@ -888,4 +1008,193 @@ mod tests { "reading a FIFO must not block" ); } + + /// Build settings with only the fields a case cares about. + fn proxy_settings(url: Option<&str>) -> UpstreamProxySettings<'_> { + UpstreamProxySettings { + url, + ..UpstreamProxySettings::default() + } + } + + #[test] + fn upstream_proxy_settings_accept_a_bare_proxy_url() { + validate_upstream_proxy_settings(&proxy_settings(Some("http://proxy.corp.com:3128"))) + .expect("a lone proxy URL is a complete configuration"); + } + + #[test] + fn upstream_proxy_settings_accept_an_empty_configuration() { + validate_upstream_proxy_settings(&UpstreamProxySettings::default()) + .expect("no proxy configured at all is valid"); + } + + #[test] + fn upstream_proxy_settings_reject_an_unsupported_scheme() { + let err = validate_upstream_proxy_settings(&proxy_settings(Some("socks5://proxy:1080"))) + .expect_err("only http:// and https:// proxies are supported"); + assert!(err.starts_with("https_proxy "), "{err}"); + assert!(err.contains("unsupported proxy scheme"), "{err}"); + } + + #[test] + fn upstream_proxy_settings_reject_inline_credentials_by_naming_the_auth_file() { + let err = validate_upstream_proxy_settings(&proxy_settings(Some("http://u:p@proxy:3128"))) + .expect_err("inline credentials would be stored in gateway config"); + assert!(err.contains("proxy_auth_file"), "{err}"); + } + + #[test] + fn upstream_proxy_settings_reject_an_empty_proxy_url() { + let err = validate_upstream_proxy_settings(&proxy_settings(Some(" "))) + .expect_err("present-but-empty is a misconfiguration, not 'unset'"); + assert_eq!(err, "https_proxy must not be empty when set"); + } + + #[test] + fn upstream_proxy_settings_reject_auxiliary_keys_without_a_proxy_url() { + // Each auxiliary key implies a proxy boundary the operator believed + // was in effect; accepting one while every dial goes direct would + // hide a fail-open state. + for (settings, key) in [ + ( + UpstreamProxySettings { + no_proxy: Some("10.0.0.0/8"), + ..UpstreamProxySettings::default() + }, + "no_proxy", + ), + ( + UpstreamProxySettings { + auth_file: Some("/etc/openshell/secrets/proxy-auth"), + ..UpstreamProxySettings::default() + }, + "proxy_auth_file", + ), + ( + UpstreamProxySettings { + connect_by_hostname: Some(true), + ..UpstreamProxySettings::default() + }, + "proxy_connect_by_hostname", + ), + ( + UpstreamProxySettings { + ca_bundle: Some("/etc/openshell/tls/proxy-ca.pem"), + ..UpstreamProxySettings::default() + }, + "proxy_ca_bundle", + ), + ] { + let err = validate_upstream_proxy_settings(&settings) + .expect_err("an auxiliary key without a proxy URL must fail closed"); + assert_eq!( + err, + format!("{key} is set but no https_proxy is configured") + ); + } + } + + #[test] + fn upstream_proxy_settings_reject_empty_auxiliary_values() { + for (settings, expected) in [ + ( + UpstreamProxySettings { + url: Some("http://proxy:3128"), + no_proxy: Some(" "), + ..UpstreamProxySettings::default() + }, + "no_proxy must not be empty when set; omit it instead", + ), + ( + UpstreamProxySettings { + url: Some("http://proxy:3128"), + auth_file: Some(""), + ..UpstreamProxySettings::default() + }, + "proxy_auth_file must not be empty when set", + ), + ( + UpstreamProxySettings { + url: Some("http://proxy:3128"), + ca_bundle: Some(""), + ..UpstreamProxySettings::default() + }, + "proxy_ca_bundle must not be empty when set", + ), + ] { + let err = validate_upstream_proxy_settings(&settings) + .expect_err("present-but-empty must never be treated as unset"); + assert_eq!(err, expected); + } + } + + #[test] + fn upstream_proxy_credentials_require_the_cleartext_acknowledgement() { + let err = validate_upstream_proxy_settings(&UpstreamProxySettings { + url: Some("http://proxy:3128"), + auth_file: Some("/etc/openshell/secrets/proxy-auth"), + ..UpstreamProxySettings::default() + }) + .expect_err("Basic auth to an http:// proxy is cleartext on the wire"); + assert!(err.contains("proxy_auth_allow_insecure"), "{err}"); + + validate_upstream_proxy_settings(&UpstreamProxySettings { + url: Some("http://proxy:3128"), + auth_file: Some("/etc/openshell/secrets/proxy-auth"), + auth_allow_insecure: Some(true), + ..UpstreamProxySettings::default() + }) + .expect("the explicit acknowledgement makes the exposure an operator decision"); + } + + #[test] + fn upstream_proxy_credentials_need_no_acknowledgement_for_an_https_proxy() { + // The credential travels inside the verified TLS session to the proxy. + validate_upstream_proxy_settings(&UpstreamProxySettings { + url: Some("https://proxy:3130"), + auth_file: Some("/etc/openshell/secrets/proxy-auth"), + ..UpstreamProxySettings::default() + }) + .expect("an https:// proxy does not expose the credential on the wire"); + + // ... but setting it anyway is tolerated rather than an error. + validate_upstream_proxy_settings(&UpstreamProxySettings { + url: Some("https://proxy:3130"), + auth_file: Some("/etc/openshell/secrets/proxy-auth"), + auth_allow_insecure: Some(true), + ..UpstreamProxySettings::default() + }) + .expect("a redundant acknowledgement is tolerated"); + } + + #[test] + fn upstream_proxy_acknowledgement_without_credentials_is_rejected() { + // Including `= false`: the operator believed an auth file was + // configured, so the mismatch is surfaced rather than ignored. + for ack in [Some(true), Some(false)] { + let err = validate_upstream_proxy_settings(&UpstreamProxySettings { + url: Some("http://proxy:3128"), + auth_allow_insecure: ack, + ..UpstreamProxySettings::default() + }) + .expect_err("the acknowledgement is meaningless without a credential"); + assert_eq!( + err, + "proxy_auth_allow_insecure is set but no proxy_auth_file is configured" + ); + } + } + + #[test] + fn upstream_proxy_ca_bundle_is_valid_with_a_plain_http_proxy() { + // A TLS-intercepting proxy can be reached over plain HTTP while still + // re-signing tunneled server certificates with its own CA. + validate_upstream_proxy_settings(&UpstreamProxySettings { + url: Some("http://proxy:3128"), + ca_bundle: Some("/etc/openshell/tls/proxy-ca.pem"), + ..UpstreamProxySettings::default() + }) + .expect("an intercepting proxy's CA is meaningful without an https:// proxy URL"); + } } diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index 4fc9ace415..df6286cfc6 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -154,6 +154,14 @@ Select the VM driver with `--drivers vm`, `OPENSHELL_DRIVERS=vm`, or `compute_dr | `guest_tls_ca` | unset | CA cert for the guest's mTLS client bundle. Required when `grpc_endpoint` uses `https://`. | | `guest_tls_cert` | unset | Guest client certificate. | | `guest_tls_key` | unset | Guest client private key. | +| `https_proxy` | unset | Corporate forward proxy (`http://host:port` or `https://host:port`) the in-guest supervisor chains policy-approved TLS CONNECT egress through. A proxy on the gateway host's loopback must be addressed as `http://host.openshell.internal:` — guest egress leaves through gvproxy, which NATs `192.168.127.254` to the host's `127.0.0.1`. | +| `no_proxy` | unset | Comma-separated bypass list for the corporate proxy only. OpenShell policy evaluation still applies. | +| `proxy_auth_file` | unset | Gateway-host path to a `user:pass` credential file. Staged root-only into the per-sandbox overlay and removed with the sandbox. | +| `proxy_auth_allow_insecure` | unset | Required with `proxy_auth_file` against an `http://` proxy: acknowledges that Basic auth is cleartext on the connection to the proxy. | +| `proxy_connect_by_hostname` | unset | Send hostnames rather than validated IPs in CONNECT. Last resort for proxies whose ACLs reject IP targets. | +| `proxy_ca_bundle` | unset | Gateway-host path to a PEM CA bundle trusted for an `https://` proxy and for certificates a TLS-intercepting proxy re-signs. | + +The proxy settings are operator-owned and deployment-level: they are not accepted through `template.driver_config.vm`, and they reach the supervisor on its command line through a per-sandbox argument file the driver writes into the overlay upperdir on every launch, so a sandbox image cannot forge or shadow them. Every present-but-invalid value is fatal at gateway or sandbox startup rather than degrading to a direct dial. See [`openshell-gateway --help`](../openshell-server/src/cli.rs) for the gateway process flag surface. diff --git a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh index 14dbc0466b..32d6ed1dff 100644 --- a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh +++ b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh @@ -192,6 +192,67 @@ prepare_guest_image_rootfs() { rm -rf "$payload_dir" } +# Driver-owned arguments appended to the supervisor's command line. +# +# The VM driver cannot build the supervisor's argv the way the container +# drivers do, so it writes the arguments it chose into the overlay upperdir +# and this script appends them verbatim. Populated by +# read_supervisor_extra_args; empty until then. +SUPERVISOR_EXTRA_ARGS=() + +# Upper bound on driver-supplied supervisor arguments. +# +# The corporate proxy settings are the only producer today and top out at ten +# entries. The cap exists so a corrupt or oversized file cannot expand into an +# unbounded command line. +SUPERVISOR_EXTRA_ARGS_MAX=32 + +read_supervisor_extra_args() { + # Read the driver-authored supervisor argument list, one argument per + # line, verbatim -- no word splitting, globbing, or expansion, so values + # containing spaces (e.g. a NO_PROXY list) survive intact. + # + # Security: this is the operator-owned egress boundary. The driver writes + # this file into the overlay upperdir on every launch, including an empty + # file when it has no arguments to pass, so the upperdir copy always + # shadows the read-only image layer. A sandbox image can therefore neither + # supply its own supervisor arguments by baking a file at this path nor + # disable the operator's by omitting one. A missing file means the driver + # passed nothing; a file it cannot read means the overlay is broken, and + # we fail closed rather than start a supervisor with a silently truncated + # egress configuration. + local args_file + args_file="$(root_path /opt/openshell/supervisor-args)" + + SUPERVISOR_EXTRA_ARGS=() + if [ ! -f "$args_file" ]; then + return 0 + fi + if [ ! -r "$args_file" ]; then + ts "FATAL: supervisor argument list ${args_file} is not readable" + exit 1 + fi + + local arg + while IFS= read -r arg; do + # render_guest_supervisor_args never emits a blank line, so one means + # the file was truncated or tampered with after the driver wrote it. + if [ -z "$arg" ]; then + ts "FATAL: empty entry in supervisor argument list" + exit 1 + fi + if [ "${#SUPERVISOR_EXTRA_ARGS[@]}" -ge "$SUPERVISOR_EXTRA_ARGS_MAX" ]; then + ts "FATAL: supervisor argument list exceeds ${SUPERVISOR_EXTRA_ARGS_MAX} entries" + exit 1 + fi + SUPERVISOR_EXTRA_ARGS+=("$arg") + done < "$args_file" + + if [ "${#SUPERVISOR_EXTRA_ARGS[@]}" -gt 0 ]; then + ts "supervisor arguments from driver: ${#SUPERVISOR_EXTRA_ARGS[@]} entries" + fi +} + exec_supervisor_in_newroot() { local chroot_bin local bootstrap="/.openshell-bootstrap" @@ -214,14 +275,16 @@ exec_supervisor_in_newroot() { "${bootstrap}/lib64/ld-linux-aarch64.so.1"; do if [ -x "/newroot${loader}" ]; then lib_path="${bootstrap}/lib:${bootstrap}/lib64:${bootstrap}/usr/lib:${bootstrap}/usr/lib64:${bootstrap}/lib/aarch64-linux-gnu:${bootstrap}/lib/x86_64-linux-gnu:${bootstrap}/usr/lib/aarch64-linux-gnu:${bootstrap}/usr/lib/x86_64-linux-gnu" - exec "$chroot_bin" /newroot "$loader" --library-path "$lib_path" "$supervisor" --workdir /sandbox + exec "$chroot_bin" /newroot "$loader" --library-path "$lib_path" \ + "$supervisor" --workdir /sandbox "${SUPERVISOR_EXTRA_ARGS[@]+"${SUPERVISOR_EXTRA_ARGS[@]}"}" fi done - exec "$chroot_bin" /newroot "$supervisor" --workdir /sandbox + exec "$chroot_bin" /newroot "$supervisor" --workdir /sandbox "${SUPERVISOR_EXTRA_ARGS[@]+"${SUPERVISOR_EXTRA_ARGS[@]}"}" fi if [ -x /newroot/opt/openshell/bin/openshell-sandbox ]; then - exec "$chroot_bin" /newroot /opt/openshell/bin/openshell-sandbox --workdir /sandbox + exec "$chroot_bin" /newroot /opt/openshell/bin/openshell-sandbox \ + --workdir /sandbox "${SUPERVISOR_EXTRA_ARGS[@]+"${SUPERVISOR_EXTRA_ARGS[@]}"}" fi done @@ -833,11 +896,13 @@ if [ -n "${OPENSHELL_SANDBOX_ID:-}" ]; then ts "OPENSHELL_SANDBOX_ID=${OPENSHELL_SANDBOX_ID}" fi +read_supervisor_extra_args + ts "starting openshell-sandbox supervisor" if [ "${ROOT_PREFIX:-}" = "/newroot" ]; then exec_supervisor_in_newroot fi -exec /opt/openshell/bin/openshell-sandbox --workdir /sandbox +exec /opt/openshell/bin/openshell-sandbox --workdir /sandbox "${SUPERVISOR_EXTRA_ARGS[@]+"${SUPERVISOR_EXTRA_ARGS[@]}"}" } if [ "${1:-}" != "--post-overlay" ]; then diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 8adcc79f92..23a3645c72 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -162,6 +162,19 @@ const GUEST_INIT_DROPIN_DIR: &str = openshell_core::container_paths::VM_GUEST_IN /// upperdir on every launch, so the image cannot forge or shadow it. const GUEST_INIT_DROPIN_MANIFEST: &str = openshell_core::container_paths::VM_GUEST_INIT_DROPIN_MANIFEST; +/// Guest path of the root-only corporate proxy credential staged by the driver. +const GUEST_UPSTREAM_PROXY_AUTH_PATH: &str = + openshell_core::container_paths::VM_GUEST_UPSTREAM_PROXY_AUTH_PATH; +/// Guest path of the corporate proxy CA bundle staged by the driver. +const GUEST_PROXY_CA_PATH: &str = openshell_core::container_paths::VM_GUEST_PROXY_CA_PATH; +/// Guest path of the driver-authored supervisor argument list. +/// +/// The counterpart of [`GUEST_INIT_DROPIN_MANIFEST`] for the supervisor's own +/// command line: written into the overlay upperdir on every launch (empty +/// when there is nothing to pass) so the guest appends exactly the arguments +/// the driver chose and a sandbox image cannot forge or shadow them. +const GUEST_SUPERVISOR_ARGS_PATH: &str = + openshell_core::container_paths::VM_GUEST_SUPERVISOR_ARGS_PATH; const IMAGE_CACHE_ROOT_DIR: &str = "images"; const IMAGE_CACHE_ROOTFS_IMAGE: &str = "rootfs.ext4"; const OVERLAY_TEMPLATE_CACHE_DIR: &str = "overlay-templates"; @@ -217,7 +230,7 @@ enum GuestImagePayloadSource { LocalDocker { rootfs_archive: PathBuf }, } -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[derive(Clone, serde::Serialize, serde::Deserialize)] pub struct VmDriverConfig { pub openshell_endpoint: String, pub state_dir: PathBuf, @@ -243,6 +256,104 @@ pub struct VmDriverConfig { /// When empty, defaults to the resolved UID. #[serde(default, skip_serializing_if = "Option::is_none")] pub sandbox_gid: Option, + + /// Corporate forward proxy URL (`http://host:port` or `https://host:port`) + /// passed to the in-guest supervisor. + /// + /// The supervisor chains policy-approved TLS tunnels through this proxy + /// with HTTP CONNECT instead of dialing destinations directly. This is an + /// operator-owned egress boundary: it travels on the supervisor's argv, + /// which sandbox spec/template environment and image `ENV` cannot + /// influence. A proxy on the gateway host's loopback is reachable from the + /// guest only through the gvproxy host alias + /// (`host.openshell.internal`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub https_proxy: Option, + + /// Comma-separated `NO_PROXY` list passed alongside the proxy URL. + /// + /// Matching destinations are dialed directly instead of through the + /// corporate proxy. This bypasses only the corporate proxy, never + /// `OpenShell` policy evaluation. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub no_proxy: Option, + + /// Path (on the gateway host) to a file containing the corporate proxy + /// credential in `user:pass` form. + /// + /// The driver validates it at sandbox-create time and stages it into the + /// per-sandbox overlay at [`GUEST_UPSTREAM_PROXY_AUTH_PATH`], root-only. + /// Credentials are never embedded in the proxy URL and never reach the + /// guest environment. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub proxy_auth_file: Option, + + /// Explicit acknowledgement that proxy credentials are sent in cleartext. + /// + /// `Proxy-Authorization: Basic` over the plain-TCP connection to an + /// `http://` proxy is recoverable by anyone on the network path, so + /// [`Self::proxy_auth_file`] requires this acknowledgement. An `https://` + /// proxy carries the credential inside the verified TLS session and does + /// not need it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub proxy_auth_allow_insecure: Option, + + /// Send the destination hostname in CONNECT requests instead of a + /// validated IP. + /// + /// The default binds the tunnel to an address that passed the sandbox's + /// SSRF and `allowed_ips` validation. Set this only when the proxy's ACLs + /// filter on hostnames and reject IP CONNECT targets: the proxy then + /// resolves the name itself and its own ACLs become the effective egress + /// control for proxied TLS. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub proxy_connect_by_hostname: Option, + + /// Path (on the gateway host) to a PEM CA bundle trusted for the + /// corporate proxy. + /// + /// The driver stages it into the per-sandbox overlay at + /// [`GUEST_PROXY_CA_PATH`] and passes that path via + /// `--upstream-proxy-ca-bundle`. It is trusted both for the handshake + /// with an `https://` proxy and for server certificates re-signed by a + /// TLS-intercepting proxy. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub proxy_ca_bundle: Option, +} + +/// Redacting `Debug` so a proxy URL or credential path never reaches a log. +/// +/// A validated proxy URL cannot embed credentials, but `Debug` can be emitted +/// before validation runs, so presence is logged rather than the value. +impl std::fmt::Debug for VmDriverConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("VmDriverConfig") + .field("openshell_endpoint", &self.openshell_endpoint) + .field("state_dir", &self.state_dir) + .field("launcher_bin", &self.launcher_bin) + .field("default_image", &self.default_image) + .field("bootstrap_image", &self.bootstrap_image) + .field("log_level", &self.log_level) + .field("krun_log_level", &self.krun_log_level) + .field("vcpus", &self.vcpus) + .field("mem_mib", &self.mem_mib) + .field("overlay_disk_mib", &self.overlay_disk_mib) + .field("guest_tls_ca", &self.guest_tls_ca) + .field("guest_tls_cert", &self.guest_tls_cert) + .field("guest_tls_key", &self.guest_tls_key) + .field("gpu_enabled", &self.gpu_enabled) + .field("gpu_mem_mib", &self.gpu_mem_mib) + .field("gpu_vcpus", &self.gpu_vcpus) + .field("sandbox_uid", &self.sandbox_uid) + .field("sandbox_gid", &self.sandbox_gid) + .field("https_proxy", &self.https_proxy.is_some()) + .field("no_proxy", &self.no_proxy) + .field("proxy_auth_file", &self.proxy_auth_file.is_some()) + .field("proxy_auth_allow_insecure", &self.proxy_auth_allow_insecure) + .field("proxy_connect_by_hostname", &self.proxy_connect_by_hostname) + .field("proxy_ca_bundle", &self.proxy_ca_bundle) + .finish() + } } /// Default sandbox UID used by the VM driver when no config value is set. @@ -269,6 +380,12 @@ impl Default for VmDriverConfig { gpu_vcpus: 4, sandbox_uid: None, sandbox_gid: None, + https_proxy: None, + no_proxy: None, + proxy_auth_file: None, + proxy_auth_allow_insecure: None, + proxy_connect_by_hostname: None, + proxy_ca_bundle: None, } } } @@ -307,6 +424,29 @@ impl VmDriverConfig { Ok(()) } + /// Validate the operator's corporate upstream-proxy settings, fail-closed. + /// + /// Delegates to the validator shared with the Podman and Kubernetes + /// drivers and with the in-guest supervisor, so a value accepted here is + /// never rejected inside the guest — and no misconfiguration can silently + /// degrade to a direct dial. + /// + /// # Errors + /// + /// Returns a message naming the offending key. + pub fn validate_proxy_config(&self) -> Result<(), String> { + openshell_core::driver_utils::validate_upstream_proxy_settings( + &openshell_core::driver_utils::UpstreamProxySettings { + url: self.https_proxy.as_deref(), + no_proxy: self.no_proxy.as_deref(), + auth_file: self.proxy_auth_file.as_deref(), + auth_allow_insecure: self.proxy_auth_allow_insecure, + connect_by_hostname: self.proxy_connect_by_hostname, + ca_bundle: self.proxy_ca_bundle.as_deref(), + }, + ) + } + fn requires_tls_materials(&self) -> bool { self.openshell_endpoint.starts_with("https://") } @@ -447,6 +587,7 @@ impl VmDriver { .validate() .map_err(|err| err.message().to_string())?; config.validate_sandbox_identity()?; + config.validate_proxy_config()?; if config.openshell_endpoint.trim().is_empty() { return Err("openshell endpoint is required".to_string()); } @@ -908,6 +1049,16 @@ impl VmDriver { return Err(err); } + // Staged on every launch, including a restart onto a preserved + // overlay, so the driver's copy always shadows the image layer. + if let Err(err) = inject_guest_upstream_proxy(&overlay_disk, &self.config).await { + self.lifecycle_extensions + .after_launch_failed(&sandbox, &state_dir, LaunchAbortReason::GuestPrepareFailed) + .await; + self.release_gpu_and_subnet(&sandbox.id); + return Err(err); + } + let endpoint_override = if plan.backend == VmBackend::Qemu { plan.host_ip.as_deref().map(|host_ip| { guest_visible_openshell_endpoint_for_tap(&self.config.openshell_endpoint, host_ip) @@ -5130,6 +5281,192 @@ fn inject_guest_init_dropins( span_status.finish(Ok(())) } +/// Build the corporate upstream-proxy arguments passed to the guest supervisor. +/// +/// This operator-owned egress boundary travels on the supervisor's argv, +/// which sandbox spec/template environment and image `ENV` cannot influence. +/// Credentials are never on argv — only the root-only guest path is passed; +/// the supervisor reads the credential from that file. +fn upstream_proxy_cli_args(config: &VmDriverConfig) -> Vec { + let mut args = Vec::new(); + if let Some(url) = &config.https_proxy { + args.push("--upstream-proxy".to_string()); + args.push(url.clone()); + } + if let Some(list) = &config.no_proxy { + args.push("--upstream-no-proxy".to_string()); + args.push(list.clone()); + } + if config.proxy_auth_file.is_some() { + args.push("--upstream-proxy-auth-file".to_string()); + // The guest path, never the gateway-host path the operator configured. + args.push(GUEST_UPSTREAM_PROXY_AUTH_PATH.to_string()); + } + // Config validation guarantees the acknowledgement is `true` whenever an + // auth file is configured against an http:// proxy; the supervisor + // independently refuses credentials without it. + if config.proxy_auth_allow_insecure == Some(true) { + args.push("--upstream-proxy-auth-allow-insecure".to_string()); + } + // Absent means the default validated-IP CONNECT binding; only the + // explicit hostname opt-in is passed through. + if config.proxy_connect_by_hostname == Some(true) { + args.push("--upstream-proxy-connect-by-hostname".to_string()); + } + if config.proxy_ca_bundle.is_some() { + args.push("--upstream-proxy-ca-bundle".to_string()); + args.push(GUEST_PROXY_CA_PATH.to_string()); + } + args +} + +/// Render the supervisor argument list as newline-separated arguments. +/// +/// One argument per line, verbatim: the guest reads the lines into an array +/// without word splitting or globbing, so values containing spaces survive +/// intact. An empty list renders an empty file, which the guest reads as "no +/// extra arguments". +fn render_guest_supervisor_args(args: &[String]) -> Vec { + let mut body = args.join("\n"); + if !body.is_empty() { + body.push('\n'); + } + body.into_bytes() +} + +/// Reject argument values the newline-delimited guest file cannot represent. +/// +/// Every value here is operator-supplied config, so this is a guard against +/// misconfiguration rather than an attack: a stray newline would otherwise +/// split one value into two arguments in the guest. +fn validate_guest_supervisor_args(args: &[String]) -> Result<(), String> { + for arg in args { + if arg.contains('\n') || arg.contains('\r') || arg.contains('\0') { + return Err( + "corporate proxy settings must not contain newline or NUL characters".to_string(), + ); + } + } + Ok(()) +} + +/// Read and validate the corporate proxy credential from the gateway host. +/// +/// Uses the validators shared with the supervisor, so a credential accepted +/// here is never rejected inside the guest. The error never carries the file +/// contents. +async fn read_sandbox_proxy_credential(path: &str) -> Result { + let path_owned = path.to_string(); + let raw = tokio::task::spawn_blocking(move || { + openshell_core::driver_utils::read_upstream_proxy_credential_file(&path_owned) + }) + .await + .map_err(|err| Status::internal(format!("proxy_auth_file read task failed: {err}")))? + .map_err(Status::invalid_argument)?; + let credential = openshell_core::driver_utils::parse_upstream_proxy_credential(&raw) + .map_err(|err| Status::invalid_argument(format!("proxy_auth_file '{path}': {err}")))?; + Ok(credential.to_string()) +} + +/// Read and validate the corporate proxy CA bundle from the gateway host. +/// +/// Checked here rather than only in the guest so the operator gets an error +/// attributable to `proxy_ca_bundle` instead of an opaque supervisor startup +/// failure inside every sandbox. +async fn read_sandbox_proxy_ca_bundle(path: &str) -> Result, Status> { + let path_owned = path.to_string(); + let bytes = tokio::task::spawn_blocking(move || fs::read(&path_owned)) + .await + .map_err(|err| Status::internal(format!("proxy_ca_bundle read task failed: {err}")))? + .map_err(|err| { + Status::invalid_argument(format!("proxy_ca_bundle '{path}' could not be read: {err}")) + })?; + if bytes.is_empty() { + return Err(Status::invalid_argument(format!( + "proxy_ca_bundle '{path}' is empty" + ))); + } + if !bytes + .windows(PEM_CERTIFICATE_MARKER.len()) + .any(|window| window == PEM_CERTIFICATE_MARKER) + { + return Err(Status::invalid_argument(format!( + "proxy_ca_bundle '{path}' contains no PEM certificate" + ))); + } + Ok(bytes) +} + +/// Marker every PEM certificate begins with, used to reject a CA bundle that +/// holds no certificate before it reaches the guest. +const PEM_CERTIFICATE_MARKER: &[u8] = b"-----BEGIN CERTIFICATE-----"; + +/// Stage the corporate upstream-proxy configuration into the guest overlay. +/// +/// Writes three files into the overlay upperdir the driver owns: +/// +/// * the credential at [`GUEST_UPSTREAM_PROXY_AUTH_PATH`], mode `0600`; +/// * the CA bundle at [`GUEST_PROXY_CA_PATH`], mode `0644` (a CA certificate +/// is not secret); +/// * the supervisor argument list at [`GUEST_SUPERVISOR_ARGS_PATH`], mode +/// `0644`. +/// +/// All three are written on every launch, empty when the corresponding +/// setting is absent. Writing rather than skipping is what makes the channel +/// unforgeable: the upperdir copy always shadows the read-only image layer, so +/// a sandbox image cannot supply its own arguments or credential by baking a +/// file at these paths, and cannot disable the operator's by omitting one. It +/// also clears material a previous launch staged into a preserved overlay +/// after the operator removed the setting. +/// +/// A microVM has no bind mounts or container secrets, so the credential lives +/// at rest inside the per-sandbox overlay disk on the host — the same +/// delivery the per-sandbox gateway JWT already uses. It is removed with the +/// sandbox when the state directory is deleted. +#[allow(clippy::result_large_err)] +async fn inject_guest_upstream_proxy( + overlay_disk: &Path, + config: &VmDriverConfig, +) -> Result<(), Status> { + // Written whether or not they are configured. Writing empty files when + // the operator removed a setting clears material a previous launch staged + // into a preserved overlay, and shadows anything an image baked at these + // paths, so a staged file is only ever the one this launch produced. + let credential = match config.proxy_auth_file.as_deref() { + Some(path) => format!("{}\n", read_sandbox_proxy_credential(path).await?).into_bytes(), + None => Vec::new(), + }; + let credential_path = overlay_upper_path(GUEST_UPSTREAM_PROXY_AUTH_PATH); + write_rootfs_image_file(overlay_disk, &credential_path, &credential) + .map_err(|err| Status::internal(format!("write VM guest proxy credential: {err}")))?; + set_rootfs_image_file_mode(overlay_disk, &credential_path, 0o600) + .map_err(|err| Status::internal(format!("set VM guest proxy credential mode: {err}")))?; + + let ca_bundle = match config.proxy_ca_bundle.as_deref() { + Some(path) => read_sandbox_proxy_ca_bundle(path).await?, + None => Vec::new(), + }; + let ca_path = overlay_upper_path(GUEST_PROXY_CA_PATH); + write_rootfs_image_file(overlay_disk, &ca_path, &ca_bundle) + .map_err(|err| Status::internal(format!("write VM guest proxy CA bundle: {err}")))?; + set_rootfs_image_file_mode(overlay_disk, &ca_path, 0o644) + .map_err(|err| Status::internal(format!("set VM guest proxy CA bundle mode: {err}")))?; + + let args = upstream_proxy_cli_args(config); + validate_guest_supervisor_args(&args).map_err(Status::failed_precondition)?; + let guest_path = overlay_upper_path(GUEST_SUPERVISOR_ARGS_PATH); + write_rootfs_image_file( + overlay_disk, + &guest_path, + &render_guest_supervisor_args(&args), + ) + .map_err(|err| Status::internal(format!("write VM guest supervisor arguments: {err}")))?; + set_rootfs_image_file_mode(overlay_disk, &guest_path, 0o644).map_err(|err| { + Status::internal(format!("set VM guest supervisor arguments mode: {err}")) + })?; + Ok(()) +} + /// Render the drop-in allow-list as newline-separated, ASCII-sorted, /// de-duplicated names. Names are already validated to be path-safe by /// [`validate_guest_init_dropins`]. @@ -8403,4 +8740,333 @@ mod tests { assert!(err.is_resource_exhausted()); assert_eq!(err.message(), "pool empty"); } + + /// A driver config carrying only corporate proxy settings. + fn proxy_config( + https_proxy: Option<&str>, + auth_file: Option<&str>, + ca_bundle: Option<&str>, + ) -> VmDriverConfig { + VmDriverConfig { + openshell_endpoint: "http://127.0.0.1:8080".to_string(), + https_proxy: https_proxy.map(ToString::to_string), + proxy_auth_file: auth_file.map(ToString::to_string), + proxy_auth_allow_insecure: auth_file.map(|_| true), + proxy_ca_bundle: ca_bundle.map(ToString::to_string), + ..Default::default() + } + } + + #[test] + fn driver_config_debug_redacts_the_proxy_url_and_credential_path() { + // `Debug` can be emitted before validation runs, and an unvalidated + // proxy URL may still carry inline `user:pass@` credentials. + let rendered = format!( + "{:?}", + proxy_config( + Some("http://user:secret@proxy.corp.test:3128"), + Some("/etc/openshell/secrets/proxy-auth"), + Some("/etc/openshell/tls/corp-ca.pem"), + ) + ); + assert!( + !rendered.contains("secret") && !rendered.contains("proxy.corp.test"), + "the proxy URL must be logged as presence only: {rendered}" + ); + assert!( + !rendered.contains("/etc/openshell/secrets/proxy-auth"), + "the credential path must be logged as presence only: {rendered}" + ); + assert!( + rendered.contains("https_proxy: true") && rendered.contains("proxy_auth_file: true"), + "presence of each must still be visible for debugging: {rendered}" + ); + // A CA path is not sensitive and stays readable. + assert!( + rendered.contains("corp-ca.pem"), + "the CA bundle path is not a secret and should stay legible: {rendered}" + ); + } + + #[test] + fn proxy_material_is_staged_inside_the_per_sandbox_overlay() { + // Everything the driver stages lands in the overlay upperdir, which + // lives in the sandbox's own state directory. That is what makes the + // credential removable with the sandbox (remove_sandbox_state_dir + // deletes the whole directory) and unforgeable by the guest image + // (the upperdir shadows the read-only image layer). + for guest_path in [ + GUEST_UPSTREAM_PROXY_AUTH_PATH, + GUEST_PROXY_CA_PATH, + GUEST_SUPERVISOR_ARGS_PATH, + ] { + assert!( + guest_path.starts_with("/opt/openshell/"), + "{guest_path} must be under the reserved guest control root" + ); + assert_eq!( + overlay_upper_path(guest_path), + format!("/upper{guest_path}"), + "{guest_path} must be staged into the overlay upperdir" + ); + } + } + + #[test] + fn upstream_proxy_args_are_empty_without_a_configured_proxy() { + assert!(upstream_proxy_cli_args(&VmDriverConfig::default()).is_empty()); + // The file is still written, empty, so the guest cannot fall back to + // an image-baked argument list. + assert!(render_guest_supervisor_args(&[]).is_empty()); + } + + #[test] + fn upstream_proxy_args_pass_guest_paths_not_host_paths() { + let config = proxy_config( + Some("http://proxy.corp.test:3128"), + Some("/etc/openshell/secrets/proxy-auth"), + Some("/etc/openshell/tls/corp-ca.pem"), + ); + let args = upstream_proxy_cli_args(&config); + + // The credential and CA live at fixed guest paths; the gateway-host + // paths the operator configured must never reach the guest argv. + let auth = args + .iter() + .position(|arg| arg == "--upstream-proxy-auth-file") + .map(|i| args[i + 1].as_str()); + assert_eq!(auth, Some(GUEST_UPSTREAM_PROXY_AUTH_PATH)); + let ca = args + .iter() + .position(|arg| arg == "--upstream-proxy-ca-bundle") + .map(|i| args[i + 1].as_str()); + assert_eq!(ca, Some(GUEST_PROXY_CA_PATH)); + assert!( + !args + .iter() + .any(|arg| arg.contains("/etc/openshell/secrets") || arg.contains("corp-ca.pem")), + "host paths leaked into the guest argv: {args:?}" + ); + } + + #[test] + fn upstream_proxy_args_pass_only_explicit_opt_ins() { + let mut config = proxy_config(Some("https://proxy.corp.test:3130"), None, None); + config.no_proxy = Some("10.0.0.0/8,.svc.cluster.local".to_string()); + let args = upstream_proxy_cli_args(&config); + assert_eq!( + args, + vec![ + "--upstream-proxy".to_string(), + "https://proxy.corp.test:3130".to_string(), + "--upstream-no-proxy".to_string(), + "10.0.0.0/8,.svc.cluster.local".to_string(), + ] + ); + + // `Some(false)` must not be passed as the presence flag it is on the + // supervisor side. + config.proxy_connect_by_hostname = Some(false); + assert!( + !upstream_proxy_cli_args(&config) + .iter() + .any(|arg| arg == "--upstream-proxy-connect-by-hostname") + ); + config.proxy_connect_by_hostname = Some(true); + assert!( + upstream_proxy_cli_args(&config) + .iter() + .any(|arg| arg == "--upstream-proxy-connect-by-hostname") + ); + } + + #[test] + fn guest_supervisor_args_render_one_argument_per_line() { + let args = vec![ + "--upstream-proxy".to_string(), + "http://proxy.corp.test:3128".to_string(), + "--upstream-no-proxy".to_string(), + "a.example, b.example".to_string(), + ]; + // A value containing a space stays one line, so the guest reads it + // back as a single argument rather than word-splitting it. + assert_eq!( + String::from_utf8(render_guest_supervisor_args(&args)).unwrap(), + "--upstream-proxy\nhttp://proxy.corp.test:3128\n--upstream-no-proxy\na.example, b.example\n" + ); + } + + #[test] + fn guest_supervisor_args_reject_line_breaking_values() { + // A newline would split one operator value into two guest arguments. + for bad in ["a\nb", "a\rb", "a\0b"] { + assert!( + validate_guest_supervisor_args(&[bad.to_string()]).is_err(), + "{bad:?} must be rejected" + ); + } + validate_guest_supervisor_args(&["--upstream-proxy".to_string()]) + .expect("ordinary arguments are accepted"); + } + + #[test] + fn proxy_config_validation_rejects_settings_without_a_proxy_url() { + let config = VmDriverConfig { + no_proxy: Some("10.0.0.0/8".to_string()), + ..Default::default() + }; + let err = config + .validate_proxy_config() + .expect_err("a bypass list without a proxy would hide a fail-open state"); + assert!(err.contains("no_proxy"), "{err}"); + + let config = proxy_config(Some("http://proxy.corp.test:3128"), None, None); + config + .validate_proxy_config() + .expect("a lone proxy URL is a complete configuration"); + } + + #[test] + fn proxy_config_validation_requires_the_cleartext_acknowledgement() { + let mut config = proxy_config( + Some("http://proxy.corp.test:3128"), + Some("/etc/openshell/secrets/proxy-auth"), + None, + ); + config.proxy_auth_allow_insecure = None; + let err = config + .validate_proxy_config() + .expect_err("Basic auth to an http:// proxy is cleartext on the wire"); + assert!(err.contains("proxy_auth_allow_insecure"), "{err}"); + } + + #[tokio::test] + async fn proxy_ca_bundle_without_a_certificate_fails_the_sandbox() { + let dir = std::env::temp_dir().join(format!("openshell-vm-ca-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("not-a-ca.pem"); + std::fs::write(&path, b"this is not a certificate\n").unwrap(); + + let err = read_sandbox_proxy_ca_bundle(path.to_str().unwrap()) + .await + .expect_err("a certificate-free bundle must fail closed"); + assert_eq!(err.code(), Code::InvalidArgument); + assert!(err.message().contains("no PEM certificate"), "{err}"); + + std::fs::write(&path, b"").unwrap(); + let err = read_sandbox_proxy_ca_bundle(path.to_str().unwrap()) + .await + .expect_err("an empty bundle must fail closed"); + assert!(err.message().contains("is empty"), "{err}"); + + let err = read_sandbox_proxy_ca_bundle(dir.join("missing.pem").to_str().unwrap()) + .await + .expect_err("an unreadable bundle must fail closed"); + assert!(err.message().contains("could not be read"), "{err}"); + + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[tokio::test] + async fn proxy_credential_is_validated_against_the_supervisor_rules() { + let dir = std::env::temp_dir().join(format!("openshell-vm-cred-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("proxy-auth"); + + std::fs::write(&path, "proxyuser:proxypass\n").unwrap(); + assert_eq!( + read_sandbox_proxy_credential(path.to_str().unwrap()) + .await + .expect("a well-formed credential is accepted"), + "proxyuser:proxypass" + ); + + // Rejected here rather than inside every sandbox's supervisor. + std::fs::write(&path, "no-separator\n").unwrap(); + let err = read_sandbox_proxy_credential(path.to_str().unwrap()) + .await + .expect_err("a malformed credential must fail closed"); + assert_eq!(err.code(), Code::InvalidArgument); + assert!( + !err.message().contains("no-separator"), + "the error must not echo credential file contents: {err}" + ); + + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn guest_environment_carries_no_corporate_proxy_settings() { + // The egress boundary is argv-only: `build_guest_environment` merges + // user-supplied environment, so anything it emitted here would be + // attacker-influenced. + let config = proxy_config( + Some("http://proxy.corp.test:3128"), + Some("/etc/openshell/secrets/proxy-auth"), + Some("/etc/openshell/tls/corp-ca.pem"), + ); + let sandbox = Sandbox { + id: "sb-proxy".to_string(), + name: "proxy".to_string(), + spec: Some(SandboxSpec { + environment: [ + ( + "HTTPS_PROXY".to_string(), + "http://attacker:3128".to_string(), + ), + ("NO_PROXY".to_string(), "*".to_string()), + ] + .into_iter() + .collect(), + ..Default::default() + }), + ..Default::default() + }; + + let env = build_guest_environment(&sandbox, &config, None); + assert!( + !env.iter().any(|entry| entry.starts_with("--upstream")), + "driver environment must never carry supervisor arguments: {env:?}" + ); + // A sandbox may still set the conventional variables for its own + // workload, but the supervisor ignores them on this path -- what + // matters is that the driver never derives the boundary from them. + assert!( + !env.iter() + .any(|entry| entry.contains("proxy.corp.test") || entry.contains("proxy-auth")), + "operator proxy settings must not reach the guest environment: {env:?}" + ); + } + + #[test] + fn sandbox_driver_config_cannot_carry_proxy_settings() { + // The upstream proxy is host network topology, not a per-sandbox + // setting: the caller-supplied envelope must reject it outright + // rather than silently ignoring it. + for key in [ + "https_proxy", + "no_proxy", + "proxy_auth_file", + "proxy_auth_allow_insecure", + "proxy_connect_by_hostname", + "proxy_ca_bundle", + ] { + let template = SandboxTemplate { + driver_config: Some(Struct { + fields: std::iter::once(( + key.to_string(), + Value { + kind: Some(Kind::StringValue("http://attacker:3128".to_string())), + }, + )) + .collect(), + }), + ..Default::default() + }; + assert!( + VmSandboxDriverConfig::from_template(&template).is_err(), + "template.driver_config.vm must reject '{key}'" + ); + } + } } diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs index 95ebf0f8b2..2546cb2606 100644 --- a/crates/openshell-driver-vm/src/main.rs +++ b/crates/openshell-driver-vm/src/main.rs @@ -146,6 +146,30 @@ struct Args { #[arg(long, env = "OPENSHELL_VM_SANDBOX_GID")] sandbox_gid: Option, + // Corporate forward proxy for sandbox egress. Operator-owned: these reach + // the guest supervisor on its argv, which the sandbox image and the + // user-supplied environment cannot influence. + #[arg(long, env = "OPENSHELL_VM_HTTPS_PROXY")] + https_proxy: Option, + + #[arg(long, env = "OPENSHELL_VM_NO_PROXY")] + no_proxy: Option, + + #[arg(long, env = "OPENSHELL_VM_PROXY_AUTH_FILE")] + proxy_auth_file: Option, + + // Value-taking rather than a presence flag so an explicit `false` in + // `[openshell.drivers.vm]` survives the gateway -> driver hop and still + // trips the "acknowledgement without a credential" check. + #[arg(long, env = "OPENSHELL_VM_PROXY_AUTH_ALLOW_INSECURE")] + proxy_auth_allow_insecure: Option, + + #[arg(long, env = "OPENSHELL_VM_PROXY_CONNECT_BY_HOSTNAME")] + proxy_connect_by_hostname: Option, + + #[arg(long, env = "OPENSHELL_VM_PROXY_CA_BUNDLE")] + proxy_ca_bundle: Option, + #[arg(long, hide = true)] vm_backend: Option, @@ -243,6 +267,12 @@ async fn main() -> Result<()> { gpu_vcpus: args.gpu_vcpus, sandbox_uid: args.sandbox_uid, sandbox_gid: args.sandbox_gid, + https_proxy: args.https_proxy.clone(), + no_proxy: args.no_proxy.clone(), + proxy_auth_file: args.proxy_auth_file.clone(), + proxy_auth_allow_insecure: args.proxy_auth_allow_insecure, + proxy_connect_by_hostname: args.proxy_connect_by_hostname, + proxy_ca_bundle: args.proxy_ca_bundle.clone(), }) .await .map_err(|err| miette::miette!("{err}"))?; @@ -620,6 +650,63 @@ mod tests { use clap::Parser; use std::path::PathBuf; + #[test] + fn corporate_proxy_flags_parse_into_driver_settings() { + let args = Args::parse_from([ + "openshell-driver-vm", + "--openshell-endpoint", + "https://host.openshell.internal:17670", + "--https-proxy", + "http://proxy.corp.com:8080", + "--no-proxy", + "10.0.0.0/8,.svc.cluster.local", + "--proxy-auth-file", + "/etc/openshell/secrets/proxy-auth", + "--proxy-auth-allow-insecure", + "true", + "--proxy-connect-by-hostname", + "false", + "--proxy-ca-bundle", + "/etc/openshell/tls/proxy-ca.pem", + ]); + + assert_eq!( + args.https_proxy.as_deref(), + Some("http://proxy.corp.com:8080") + ); + assert_eq!( + args.no_proxy.as_deref(), + Some("10.0.0.0/8,.svc.cluster.local") + ); + assert_eq!( + args.proxy_auth_file.as_deref(), + Some("/etc/openshell/secrets/proxy-auth") + ); + assert_eq!(args.proxy_auth_allow_insecure, Some(true)); + // Value-taking rather than a presence flag, so the gateway can + // forward an explicit `false` from `[openshell.drivers.vm]`. + assert_eq!(args.proxy_connect_by_hostname, Some(false)); + assert_eq!( + args.proxy_ca_bundle.as_deref(), + Some("/etc/openshell/tls/proxy-ca.pem") + ); + } + + #[test] + fn corporate_proxy_settings_default_to_unset() { + let args = Args::parse_from([ + "openshell-driver-vm", + "--openshell-endpoint", + "https://host.openshell.internal:17670", + ]); + assert!(args.https_proxy.is_none()); + assert!(args.no_proxy.is_none()); + assert!(args.proxy_auth_file.is_none()); + assert!(args.proxy_auth_allow_insecure.is_none()); + assert!(args.proxy_connect_by_hostname.is_none()); + assert!(args.proxy_ca_bundle.is_none()); + } + #[test] fn peer_authorization_accepts_matching_uid_and_pid() { authorize_peer_credentials( diff --git a/crates/openshell-gateway/src/vm.rs b/crates/openshell-gateway/src/vm.rs index e86de28c12..f52bd9ddfa 100644 --- a/crates/openshell-gateway/src/vm.rs +++ b/crates/openshell-gateway/src/vm.rs @@ -99,6 +99,34 @@ pub struct VmComputeConfig { /// Host-side private key for the guest's mTLS client bundle. pub guest_tls_key: Option, + + /// Corporate forward proxy URL (`http://host:port` or `https://host:port`) + /// for policy-approved TLS egress from VM sandboxes. + /// + /// Deployment-level configuration, not a per-sandbox setting: it is passed + /// to the driver, which puts it on the guest supervisor's argv. A proxy on + /// this host's loopback is reachable from a guest only through the gvproxy + /// host alias `host.openshell.internal`. + pub https_proxy: Option, + + /// Comma-separated `NO_PROXY` list. Bypasses only the corporate proxy, + /// never `OpenShell` policy evaluation. + pub no_proxy: Option, + + /// Path on this host to a `user:pass` corporate proxy credential file. + pub proxy_auth_file: Option, + + /// Acknowledgement that Basic auth to an `http://` proxy is cleartext. + /// Required alongside `proxy_auth_file` unless the proxy is `https://`. + pub proxy_auth_allow_insecure: Option, + + /// Send hostnames rather than validated IPs in CONNECT requests. Last + /// resort for proxies whose ACLs reject IP CONNECT targets. + pub proxy_connect_by_hostname: Option, + + /// Path on this host to a PEM CA bundle trusted for the corporate proxy + /// and for server certificates a TLS-intercepting proxy re-signs. + pub proxy_ca_bundle: Option, } impl VmComputeConfig { @@ -135,6 +163,29 @@ impl VmComputeConfig { 4096 } + /// Validate the corporate upstream-proxy settings, fail-closed. + /// + /// Runs in the gateway as well as in the driver so an invalid + /// `[openshell.drivers.vm]` table reports the offending key instead of + /// surfacing as an opaque driver-startup timeout. + /// + /// # Errors + /// + /// Returns a [`Error::config`] naming the offending key. + pub fn validate_proxy_config(&self) -> Result<()> { + openshell_core::driver_utils::validate_upstream_proxy_settings( + &openshell_core::driver_utils::UpstreamProxySettings { + url: self.https_proxy.as_deref(), + no_proxy: self.no_proxy.as_deref(), + auth_file: self.proxy_auth_file.as_deref(), + auth_allow_insecure: self.proxy_auth_allow_insecure, + connect_by_hostname: self.proxy_connect_by_hostname, + ca_bundle: self.proxy_ca_bundle.as_deref(), + }, + ) + .map_err(Error::config) + } + #[must_use] fn default_driver_search_dirs(home: Option) -> Vec { let mut dirs = Vec::new(); @@ -163,6 +214,12 @@ impl Default for VmComputeConfig { guest_tls_ca: None, guest_tls_cert: None, guest_tls_key: None, + https_proxy: None, + no_proxy: None, + proxy_auth_file: None, + proxy_auth_allow_insecure: None, + proxy_connect_by_hostname: None, + proxy_ca_bundle: None, } } } @@ -460,6 +517,8 @@ pub async fn spawn( )); } + vm_config.validate_proxy_config()?; + let driver_bin = resolve_compute_driver_bin(vm_config)?; let socket_path = compute_driver_socket_path(vm_config); let guest_tls_paths = compute_driver_guest_tls_paths(vm_config)?; @@ -501,6 +560,7 @@ pub async fn spawn( command.arg("--guest-tls-cert").arg(tls.cert); command.arg("--guest-tls-key").arg(tls.key); } + append_upstream_proxy_args(&mut command, vm_config); let mut child = command.spawn().map_err(|e| { Error::execution(format!( @@ -515,6 +575,38 @@ pub async fn spawn( )) } +/// Forward the operator's corporate proxy settings to the driver subprocess. +/// +/// Only keys the operator actually set are passed, so the driver keeps the +/// same "omitted means no proxy" contract the supervisor enforces. The +/// booleans travel as explicit values rather than presence flags so an +/// explicit `false` still trips the driver's pairing checks. +#[cfg(unix)] +fn append_upstream_proxy_args(command: &mut Command, vm_config: &VmComputeConfig) { + if let Some(url) = &vm_config.https_proxy { + command.arg("--https-proxy").arg(url); + } + if let Some(list) = &vm_config.no_proxy { + command.arg("--no-proxy").arg(list); + } + if let Some(path) = &vm_config.proxy_auth_file { + command.arg("--proxy-auth-file").arg(path); + } + if let Some(allow) = vm_config.proxy_auth_allow_insecure { + command + .arg("--proxy-auth-allow-insecure") + .arg(allow.to_string()); + } + if let Some(by_hostname) = vm_config.proxy_connect_by_hostname { + command + .arg("--proxy-connect-by-hostname") + .arg(by_hostname.to_string()); + } + if let Some(path) = &vm_config.proxy_ca_bundle { + command.arg("--proxy-ca-bundle").arg(path); + } +} + #[cfg(unix)] fn append_otlp_args(command: &mut Command, otlp_config: Option<&OtlpConfig>, gateway_name: &str) { if let Some(config) = otlp_config { @@ -606,9 +698,10 @@ async fn connect_compute_driver(socket_path: &Path) -> Result { #[cfg(all(test, unix))] mod tests { use super::{ - VmComputeConfig, append_otlp_args, compute_driver_guest_tls_paths, - compute_driver_socket_path, current_euid, prepare_compute_driver_socket_path, - prepare_vm_state_dir, resolve_compute_driver_bin, resolve_driver_search_dirs, + VmComputeConfig, append_otlp_args, append_upstream_proxy_args, + compute_driver_guest_tls_paths, compute_driver_socket_path, current_euid, + prepare_compute_driver_socket_path, prepare_vm_state_dir, resolve_compute_driver_bin, + resolve_driver_search_dirs, }; use openshell_server::config_file::OtlpConfig; use std::os::unix::fs::PermissionsExt; @@ -644,6 +737,83 @@ mod tests { ); } + #[test] + fn vm_driver_command_forwards_corporate_proxy_settings() { + let mut command = tokio::process::Command::new("openshell-driver-vm"); + append_upstream_proxy_args( + &mut command, + &VmComputeConfig { + https_proxy: Some("http://proxy.corp.com:8080".to_string()), + no_proxy: Some("10.0.0.0/8".to_string()), + proxy_auth_file: Some("/etc/openshell/secrets/proxy-auth".to_string()), + proxy_auth_allow_insecure: Some(true), + proxy_connect_by_hostname: Some(false), + proxy_ca_bundle: Some("/etc/openshell/tls/proxy-ca.pem".to_string()), + ..VmComputeConfig::default() + }, + ); + + let args = command + .as_std() + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect::>(); + assert_eq!( + args, + [ + "--https-proxy", + "http://proxy.corp.com:8080", + "--no-proxy", + "10.0.0.0/8", + "--proxy-auth-file", + "/etc/openshell/secrets/proxy-auth", + "--proxy-auth-allow-insecure", + "true", + // Passed as an explicit value, not a presence flag, so the + // driver still sees the operator's `false`. + "--proxy-connect-by-hostname", + "false", + "--proxy-ca-bundle", + "/etc/openshell/tls/proxy-ca.pem", + ] + ); + } + + #[test] + fn vm_driver_command_omits_unset_corporate_proxy_settings() { + let mut command = tokio::process::Command::new("openshell-driver-vm"); + append_upstream_proxy_args(&mut command, &VmComputeConfig::default()); + assert_eq!(command.as_std().get_args().count(), 0); + } + + #[test] + fn invalid_corporate_proxy_config_is_rejected_before_the_driver_starts() { + // Without this the operator would see an opaque driver-readiness + // timeout instead of an error naming the offending key. + let err = VmComputeConfig { + https_proxy: Some("socks5://proxy.corp.com:1080".to_string()), + ..VmComputeConfig::default() + } + .validate_proxy_config() + .expect_err("only http:// and https:// proxies are supported"); + assert!(err.to_string().contains("https_proxy"), "{err}"); + + let err = VmComputeConfig { + proxy_ca_bundle: Some("/etc/openshell/tls/proxy-ca.pem".to_string()), + ..VmComputeConfig::default() + } + .validate_proxy_config() + .expect_err("a CA bundle without a proxy URL would hide a fail-open state"); + assert!(err.to_string().contains("proxy_ca_bundle"), "{err}"); + + VmComputeConfig { + https_proxy: Some("http://proxy.corp.com:8080".to_string()), + ..VmComputeConfig::default() + } + .validate_proxy_config() + .expect("a lone proxy URL is a complete configuration"); + } + #[test] fn resolve_driver_bin_uses_driver_dir_when_binary_present() { let dir = tempdir().unwrap(); diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index c04c0040d0..b4dc6d68fe 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -796,6 +796,43 @@ guest_tls_key = "/var/lib/openshell/guest-tls/client-key.pem" # Defaults to 10001 when unset; matching GID is used if sandbox_gid is empty. # Any non-root Linux UID/GID is valid. # sandbox_uid = 20001 +# Corporate forward proxy for sandbox egress. The keys, their semantics, and +# the fail-closed contract are identical to the Podman driver above: only TLS +# (CONNECT) egress is chained, plain-HTTP destination requests always dial +# directly, credentials must come from proxy_auth_file rather than the URL, +# an http:// proxy with credentials requires proxy_auth_allow_insecure, and +# any present-but-invalid value is rejected at gateway startup rather than +# degrading to a direct dial. proxy_auth_file and proxy_ca_bundle are paths on +# the gateway host. +# +# The sandbox cannot select or override these settings. They reach the guest +# supervisor on its command line through a per-sandbox file the driver writes +# into the overlay upperdir on every launch, so a sandbox image cannot supply +# its own values or disable the operator's by baking a file at that path. +# +# Reachability: guest egress leaves through gvproxy, so a proxy listening on +# the gateway host's loopback is reachable only through the host alias +# host.openshell.internal (gvproxy NATs 192.168.127.254 to the host's +# 127.0.0.1). Address it as http://host.openshell.internal:, not +# http://127.0.0.1:. A proxy on the corporate network is dialed out +# through the host process and needs no alias. The guest's callback to the +# gateway is unaffected and never traverses the proxy. +# +# Because a microVM has no bind mounts or container secrets, the driver stages +# the credential and the CA into the per-sandbox overlay disk: the credential +# root-only inside the guest, and both removed with the sandbox. The +# credential is therefore at rest in that overlay image on the gateway host — +# the same delivery the per-sandbox gateway token already uses, and a +# difference from the Podman secret model worth noting when choosing where to +# keep proxy credentials. +# https_proxy = "http://host.openshell.internal:8080" +# no_proxy = "10.0.0.0/8,.internal.example" +# proxy_auth_file = "/etc/openshell/secrets/proxy-auth" +# proxy_auth_allow_insecure = true +# Last resort for hostname-filtering proxy ACLs; see the Podman section above. +# proxy_connect_by_hostname = true +# Corporate CA trusted for an https:// proxy and TLS-intercepting proxies. +# proxy_ca_bundle = "/etc/openshell/tls/proxy-ca.pem" ``` ### Extension Driver diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index f00c49afd6..dbfc08c3ec 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -360,6 +360,16 @@ The VM driver creates nftables rules on the host for each sandbox VM's TAP netwo On hosts with restrictive firewalls (e.g. firewalld), the host firewall may additionally block VM traffic that the driver's rules accept. If VM sandboxes cannot reach the network, verify that the host firewall allows forwarding and input for `vmtap-*` interfaces. See the [VM driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-vm/README.md#host-side-nftables-rules) for details. +### Corporate Proxy Egress + +For proxy-required networks, the VM driver accepts the same corporate egress proxy keys as the Podman driver: `https_proxy`, `no_proxy`, `proxy_auth_file`, `proxy_auth_allow_insecure`, `proxy_connect_by_hostname`, and `proxy_ca_bundle`. The in-guest supervisor chains policy-approved TLS tunnels through the proxy with HTTP CONNECT instead of dialing destinations directly. + +The settings reach the guest supervisor on its command line through a per-sandbox argument file the driver writes into the overlay upperdir on every launch, so a sandbox cannot select, alter, or disable the proxy from inside the guest — including through image `ENV`, the sandbox environment, or files baked into the image at the paths the driver uses. + +Guest egress leaves through gvproxy, so a proxy listening on the gateway host's loopback is reachable only through the host alias: configure `https_proxy = "http://host.openshell.internal:"` rather than a `127.0.0.1` URL. A proxy on the corporate network needs no alias, and the guest's callback to the gateway never traverses the proxy. + +Because a microVM has no bind mounts or container secrets, the driver stages the credential (root-only) and the CA bundle into the per-sandbox overlay disk and removes them with the sandbox. See the [Gateway Configuration File](./gateway-config) reference for the full contract, including the cleartext-credential acknowledgement and the validated-IP CONNECT behavior. + ## Kubernetes Driver Kubernetes-backed sandboxes run as pods in the configured sandbox namespace. Use Kubernetes for shared clusters, remote compute, GPU scheduling, and operator-managed environments. diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 6dbf46a392..18556d0f7b 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -108,6 +108,11 @@ name = "vm_gateway_start" path = "tests/vm_gateway_start.rs" required-features = ["e2e-vm"] +[[test]] +name = "vm_corporate_proxy" +path = "tests/vm_corporate_proxy.rs" +required-features = ["e2e-vm"] + [[test]] name = "provider_token_exchange" path = "tests/provider_token_exchange.rs" diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index 9acc633d65..96da2a879f 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -409,4 +409,5 @@ else run_e2e_test host_gateway_alias run_e2e_test vm_overlay run_e2e_test vm_gateway_start + run_e2e_test vm_corporate_proxy fi diff --git a/e2e/rust/src/harness/host_process.rs b/e2e/rust/src/harness/host_process.rs new file mode 100644 index 0000000000..f9fefd75a6 --- /dev/null +++ b/e2e/rust/src/harness/host_process.rs @@ -0,0 +1,103 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Host-process TCP fixtures for e2e tests. +//! +//! [`HostSupportContainer`](super::container::HostSupportContainer) publishes +//! the same shape of fixture through a container engine. VM sandboxes reach +//! the host through gvproxy's `host.openshell.internal` alias and the VM e2e +//! lane has no container runtime of its own, so this variant runs the fixture +//! as a plain host process instead — keeping the lane free of a container +//! dependency it does not otherwise need. + +use std::io::Read as _; +use std::path::PathBuf; +use std::process::{Child, Command, Stdio}; +use std::time::Duration; + +use super::port::wait_for_port; + +/// A `python3` fixture listening on a host TCP port. +/// +/// Output is captured to a temp file rather than a pipe: these fixtures log +/// every request they serve, and a full pipe buffer would block the process +/// mid-test. [`logs`](Self::logs) reads the file, which is where a test finds +/// its evidence (the CONNECT targets a proxy saw, for example). +pub struct HostPythonFixture { + /// Host port the fixture listens on. + pub port: u16, + child: Child, + log_path: PathBuf, +} + +impl HostPythonFixture { + /// Start `python3 -c